def prologue(self):
        message.message.verbosity(message.INT_DEBUG)
        message.warning('a warning %(c)d', c=666)

        import os, signal
        message.debug(str(os.getpid()))
        os.kill(os.getpid(), signal.SIGSEGV)
 def prologue(self) :
   message.message.verbosity(message.INT_DEBUG)
   message.warning('a warning %(c)d', c=666)
   
   import os, signal
   message.debug(str(os.getpid()))
   os.kill(os.getpid(), signal.SIGSEGV)
 def close(self):
     table = 'goal' if self.reference else 'hits'
     if len(self.data) == 0:
         message.note('No data to upload into table "%(table)s", skipping',
                      table=table)
         return
     message.information(
         'starting data upload to table "%(table)s" via insert',
         table=table)
     with mdb.mdb.cursor() as cursor:
         rows = cursor.executemany(
             'INSERT INTO ' + table + ' (log_id, bucket_id, ' + table +
             ') VALUES (%s, %s, %s);', self.data)
         warnings = cursor.warning_count()
     if warnings:
         message.warning('upload to db via insert with %(warnings)',
                         warnings=warnings)
     if rows is None:
         message.warning(
             'upload to db via insert "%(table)s" returned None',
             table=table)
     else:
         message.information(
             'upload to db via insert added %(rows)d rows of %(data)d to "%(table)s"',
             rows=int(rows),
             data=len(self.data),
             table=table)
Beispiel #4
0
    def get_all_assets(self):
        (css_list, js_list) = ([], [])
        strip = lambda _: _.strip()
        parser = self._get_parser()
        section = 'Framework'
        if parser.has_option(section, 'user_css'):
            value = parser.get(section, 'user_css')
            css_list = value.split(' ') if value else []

        if parser.has_option(section, 'user_js'):
            value = parser.get(section, 'user_js')
            js_list = value.split(' ') if value else []

        if parser.has_option(section, 'libs'):
            value = parser.get(section, 'libs')
            if value:
                libs = set(filter(bool, map(strip, value.split(' '))))
                dependencies = set()
                for lib in libs:
                    self.get_all_dependencies(lib, dependencies)
                for name in dependencies:
                    if name not in self.libs:
                        warning("%s lib doesn't exist!" % name)
                        continue
                    group = self.libs[name]
                    if 'js' in group:
                        js_list += group['js']
                    if 'css' in group:
                        css_list += group['css']
        js_list = set(filter(bool, map(strip, js_list)))
        css_list = set(filter(bool, map(strip, css_list)))
        return (css_list, js_list)
Beispiel #5
0
 def connect(self, *args, **kwargs):
     try:
         self.db = kwargs['db']
     except KeyError:
         self.db = self.default_db
     try:
         instance = MySQLdb.connect(host=self.default_host,
                                    port=self.default_port,
                                    db=self.default_db,
                                    user=self.default_user,
                                    passwd=self.default_passwd)
         instance.autocommit(False)
     except:
         message.warning(
             'Unable to connect to mysql db %(db)s at %(host)s:%(port)d because %(exc)s',
             db=self.db,
             host=self.default_host,
             port=self.default_port,
             exc=sys.exc_info()[0])
         return
     message.note(
         "Connected to mysql db %(db)s at %(host)s:%(port)d for %(thread)s",
         db=self.default_db,
         host=self.default_host,
         port=self.default_port,
         thread=self.id())
     # this should be keyed on db too - but we don't used multiple databases currently
     self.instance[self.id()] = instance
Beispiel #6
0
 def success(self):
     'Generic success hook'
     if self.is_success is not None:
         message.warning(
             'success() called after test issue (status : %(status)s)',
             status=str(self.is_success))
     self.is_success = True
     self.SUCCESS()
 def coverage(self, pad=True, cursor=True) :
   with mdb.connection().row_cursor() if cursor else mdb.connection().cursor() as db :
     db.execute(self.query % self.__dict__)
     for result in db.fetchall() :
       yield result
     while (pad) : 
       message.warning('missing bucket')
       yield {}
Beispiel #8
0
 def plusarg_opt_int(cls, name, default, fmt='08x') :
   'To get default/command line options'
   try :
     result = int(plusargs().get(name, str(default)), 0)
   except :
     message.warning(str(sys.exc_info()))
     result = default
   message.information('Using %(result)'+fmt+' for option "%(name)s"', result=result, name=name)
   return result
 def coverage(self, pad=True, cursor=True):
     with mdb.connection().row_cursor() if cursor else mdb.connection(
     ).cursor() as db:
         db.execute(self.query % self.__dict__)
         for result in db.fetchall():
             yield result
         while (pad):
             message.warning('missing bucket')
             yield {}
 def number_of_axes(self, max_axes) :
   limit = len(self.NAMES)
   if max_axes < 1 :
     result = 1
   elif max_axes > limit : 
     result = limit
     message.warning('limit of %(limit)d axes', limit=limit)
   else :
     result = max_axes
   return result+1
 def number_of_axes(self, max_axes):
     limit = len(self.NAMES)
     if max_axes < 1:
         result = 1
     elif max_axes > limit:
         result = limit
         message.warning('limit of %(limit)d axes', limit=limit)
     else:
         result = max_axes
     return result + 1
Beispiel #12
0
 def summary(self, verbose=False) :
   is_root = mdb_conn.is_root()
   results = database.rgr().result(mdb_conn.log_id, is_root)
   result = results.summary()
   if result.passes != result.total :
     if is_root or verbose :
       for test in results[1:] : # drop this
         if test.status.status is not 'PASS' :
           message.warning("[%(log_id)d, %(status)s] %(reason)s", log_id=test.log.log_id, **test.status)
   result.summary()
   return result
Beispiel #13
0
 def plusarg_opt_int(cls, name, default, fmt='08x'):
     'To get default/command line options'
     try:
         result = int(plusargs().get(name, str(default)), 0)
     except:
         message.warning(str(sys.exc_info()))
         result = default
     message.information('Using %(result)' + fmt + ' for option "%(name)s"',
                         result=result,
                         name=name)
     return result
Beispiel #14
0
 def connect(self, *args, **kwargs) :
   try :
     self.db = kwargs['db']
   except KeyError:
     self.db = self.default_db
   try :
     instance = sqlite3.connect(self.db)
   except :
     message.warning('Unable to connect. File %(db)s because %(exc)s', db=self.db, exc=sys.exc_info()[0])
   instance.execute('PRAGMA journal_mode=WAL;')
   instance.execute('PRAGMA read_uncommitted = 1;')
   self.instance[self.id()] = instance
 def enqueue(cmd):
     'just execute here'
     message.debug('enqueue %(cmd)s', cmd=cmd)
     result = subprocess.Popen(
         cmd.split(' '),
         env=dict(os.environ,
                  MDB='root=' + str(self.mdb.get_root()) + ',parent=' +
                  str(self.mdb.log_id))).wait()
     if result > 0:
         message.warning('process %(cmd)s returned non zero %(result)d',
                         cmd=cmd,
                         result=result)
Beispiel #16
0
 def summary(self, verbose=False):
     is_root = mdb_conn.is_root()
     results = database.rgr().result(mdb_conn.log_id, is_root)
     result = results.summary()
     if result.passes != result.total:
         if is_root or verbose:
             for test in results[1:]:  # drop this
                 if test.status.status is not 'PASS':
                     message.warning("[%(log_id)d, %(status)s] %(reason)s",
                                     log_id=test.log.log_id,
                                     **test.status)
     result.summary()
     return result
Beispiel #17
0
 def __init__(self, log_ids=[], test_ids=[], xml=None, threshold=0, robust=False, previous=None) :
   'log_ids is a list of regression roots'
   self.log_ids = log_ids
   s_log_ids = ','.join(map(str, log_ids))
   self.tests = mdb.connection().row_cursor()
   if log_ids :
     # create table of individual runs, but not root node as this may have already summarised coverage
     self.tests.execute('CREATE TEMPORARY TABLE '+self.invs+' AS SELECT l1.*, goal_id AS master FROM log AS l0 JOIN log AS l1 ON (l0.log_id = l1.root) LEFT OUTER JOIN master ON (l1.log_id = master.log_id) WHERE l1.root IN ('+s_log_ids+');')
     self.tests.execute('SELECT count(*) AS children FROM '+self.invs)
     children = self.tests.fetchone().children
     if children :
       message.information('%(log_ids)s %(has)s %(children)d children', log_ids=s_log_ids, children=children, has='have' if len(log_ids) > 1 else 'has')
   # append individual runs as given by test_ids
   if xml :
     xml_ids = xml.xml.xpath('/optimize/test/log_id/text()')
   else :
     xml_ids=[]
   if test_ids or xml_ids :
     s_test_ids = ','.join(map(str, test_ids+xml_ids))
     create = ('INSERT INTO '+self.invs) if log_ids else ('CREATE TEMPORARY TABLE '+self.invs+' AS')
     self.tests.execute(create+' SELECT log.*, IFNULL(goal_id, goal.log_id) AS master FROM log LEFT OUTER JOIN master ON (log.log_id = master.log_id) LEFT OUTER JOIN goal ON (log.log_id = goal.log_id) WHERE log.log_id IN ('+s_test_ids+') GROUP BY log_id;')
   self.tests.execute('SELECT count(*) AS tests FROM '+self.invs)
   tests = self.tests.fetchone().tests
   if tests < 1 :
     message.fatal('no tests')
   message.information('starting with %(count)d tests in table %(table)s', count=tests, table=self.invs)
   # check congruency
   self.cvg = mdb.connection().row_cursor()
   rows=self.cvg.execute("SELECT md5_self AS md5, 'md5_self' AS type, invs.master, invs.root FROM point JOIN "+self.invs+" AS invs ON (invs.master = point.log_id AND point.parent IS NULL) GROUP BY md5;")
   md5 = self.cvg.fetchall()
   if not md5 :
     message.fatal('no master')
   elif len(md5) > 1 :
     message.fatal('md5 of multiple masters do not match')
   else :
     message.debug('md5 query returns %(rows)d', rows=rows)
   self.master = mdb.accessor(md5=md5[0])
   self.cvg.execute("SELECT DISTINCT(md5_axes) AS md5, 'md5_axes' AS type, invs.master, invs.root FROM point JOIN "+self.invs+" AS invs ON (invs.master = point.log_id AND point.parent IS NULL) GROUP BY md5;")
   md5 = self.cvg.fetchall()
   if len(md5) > 1 :
     message.fatal('md5 of multiple axis masters do not match')
   self.master.axes = md5[0]
   # create status table, collating goal & hits
   self.cvg.execute('CREATE TEMPORARY TABLE '+self.covg+' (bucket_id INTEGER NOT NULL PRIMARY KEY, goal INTEGER, hits INTEGER, total_hits INTEGER, rhits INTEGER, max_hits INTEGER, tests INTEGER);')
   try :
     self.threshold = float(threshold)
   except :
     self.threshold = 0.0
     message.warning('cannot convert threshold value given "%(arg)s" to float because %(exception)s, using %(threshold)2.1f', arg=threshold, exception=sys.exc_info()[0], threshold=self.threshold)
   self.robust = robust
   self.previous = previous
Beispiel #18
0
 def retry(self, fn0, fn1=None) :
   for attempt in range(0, connection.RETRIES) :
     try :
       return fn0()
     except MySQLdb.OperationalError :
       if sys.exc_info()[1].args == (2006, 'MySQL server has gone away') :
         message.warning('MySQL connection lost; retrying')
         self.reconnect()
         if fn1 :
           fn1()
       else :
         raise
   message.warning('retried %(n)d times', n=connection.RETRIES)
   raise
Beispiel #19
0
 def enqueue(self, cmd):
     'just execute here'
     message.debug('enqueue %(cmd)s', cmd=cmd)
     result = subprocess.Popen(
         cmd.split(' '),
         env=dict(os.environ,
                  MDB='root=' + str(mdb_conn.get_root()) + ',parent=' +
                  str(mdb_conn.log_id),
                  PYTHONPATH=os.environ['PYTHONPATH'] +
                  ':../python')).wait()
     if result > 0:
         message.warning('process %(cmd)s returned non zero %(result)d',
                         cmd=cmd,
                         result=result)
Beispiel #20
0
 def cb_fn(self) :
   'As callback executes vpi_get_value, could change callback fn and cb_filter signature to fn(value)'
   self.cnt += 1
   if self.cb_filter() :
     self.filtered += 1
     return
   for func in self.funcs :
     try :
       func()
     except Exception, exc:
       self.excepted += 1
       message.error('Exception in callback "%(name)s" : %(exc)s', exc=str(exc), name=self.name)
       for detail in traceback.format_exc().strip('\n').split('\n') :
         message.warning(detail)
Beispiel #21
0
 def connect(self, *args, **kwargs):
     try:
         self.db = kwargs['db']
     except KeyError:
         self.db = self.default_db
     try:
         instance = sqlite3.connect(self.db)
     except:
         message.warning('Unable to connect. File %(db)s because %(exc)s',
                         db=self.db,
                         exc=sys.exc_info()[0])
     instance.execute('PRAGMA journal_mode=WAL;')
     instance.execute('PRAGMA read_uncommitted = 1;')
     self.instance[self.id()] = instance
Beispiel #22
0
 def retry(self, fn0, fn1=None):
     for attempt in range(0, connection.RETRIES):
         try:
             return fn0()
         except MySQLdb.OperationalError:
             if sys.exc_info()[1].args == (2006,
                                           'MySQL server has gone away'):
                 message.warning('MySQL connection lost; retrying')
                 self.reconnect()
                 if fn1:
                     fn1()
             else:
                 raise
     message.warning('retried %(n)d times', n=connection.RETRIES)
     raise
Beispiel #23
0
 def close(self) :
   table = 'goal' if self.reference else 'hits'
   if len(self.data) == 0 :
     message.note('No data to upload into table "%(table)s", skipping', table=table)
     return
   message.information('starting data upload to table "%(table)s" via insert', table=table)
   with mdb.mdb.cursor() as cursor :
     rows = cursor.executemany('INSERT INTO '+table+' (log_id, bucket_id, '+table+') VALUES (%s, %s, %s);', self.data)
     warnings = cursor.warning_count()
   if warnings :
     message.warning('upload to db via insert with %(warnings)', warnings=warnings)
   if rows is None :
     message.warning('upload to db via insert "%(table)s" returned None', table=table)
   else :
     message.information('upload to db via insert added %(rows)d rows of %(data)d to "%(table)s"', rows=int(rows), data=len(self.data), table=table)
Beispiel #24
0
 def connect(self, *args, **kwargs) :
   try :
     self.db = kwargs['db']
   except KeyError:
     self.db = self.default_db
   try :
     instance = MySQLdb.connect(
       host=self.default_host,
       port=self.default_port,
       db=self.default_db,
       user=self.default_user,
       passwd=self.default_passwd
     )
     instance.autocommit(False)
   except :
     message.warning('Unable to connect to mysql db %(db)s at %(host)s:%(port)d because %(exc)s', db=self.db, host=self.default_host, port=self.default_port, exc=sys.exc_info()[0])
     return
   message.note("Connected to mysql db %(db)s at %(host)s:%(port)d for %(thread)s", db=self.default_db, host=self.default_host, port=self.default_port, thread=self.id())
   # this should be keyed on db too - but we don't used multiple databases currently
   self.instance[self.id()] = instance
Beispiel #25
0
def _set(paths):
    """ Loads a Set from one or more YAML files.

    Parameters
        paths
            iterable having the file paths to be loaded.
    Raises
        IOError
            if one of the files could not be read.
        YAMLError
            if one of the files is not a valid YAML file.
        validate.SemanticError
            if one of the files is semantically invalid.
    Returns
        Set containing all units found in the specified files.
    """

    units = Set()
    groups = {}
    virtuals = {}
    registry = Registry()

    for path in paths:
        try:
            definition = yaml(path)
            validate.set(definition)
        except IOError:
            raise
        except YAMLError:
            raise
        except validate.SemanticError:
            raise

        repository = findall('([a-zA-Z0-9]+)', path)[-3]

        for name in definition.iterkeys():
            for version in definition[name].iterkeys():
                for architecture in definition[name][version].iterkeys():
                    data = definition[name][version][architecture]
                    package = Package(name, version, architecture, repository,
                                      data)

                    try:
                        registry.add_package(name, version, architecture)
                    except PackageInRegistry:
                        warning(
                            "duplicate package found: {0}:{1} {2} from repository '{3}'. Ignoring."
                            .format(name, architecture, version, repository))
                        break
                    except GroupInRegistry:
                        warning(
                            "name conflict between group {0} from repository '{3}' and package {0}:{1} {2}. Ignoring."
                            .format(name, architecture, version, repository))
                        break
                    except VirtualPackageInRegistry:
                        warning(
                            "name conflict between virtual package {0} from repository '{3}' and package {0}:{1} {2}. Ignoring."
                            .format(name, architecture, version, repository))
                        break

                    if data['provides'] is not None:
                        for virtual in data['provides']:
                            try:
                                registry.add_virtual(virtual)
                            except PackageInRegistry:
                                warning(
                                    "name conflict between virtual package {0} from repository '{3}' and package {0}:{1} {2}. Ignoring."
                                    .format(name, architecture, version,
                                            repository))
                                break
                            except GroupInRegistry:
                                warning(
                                    "name conflict between virtual package {0} from repository '{1}' and group {0}. Ignoring."
                                    .format(virtual, repository))
                                break

                            try:
                                virtuals[virtual].provided_by(package)
                            except KeyError:
                                virtuals[virtual] = VirtualPackage(virtual)
                                virtuals[virtual].provided_by(package)

                    if data['groups'] is not None:
                        for group in data['groups']:
                            try:
                                registry.add_group(group)
                            except VirtualPackageInRegistry:
                                warning(
                                    "name conflict between group {0} from repository '{1}' and virtual package {0}. Ignoring."
                                    .format(group, repository))
                                break
                            except PackageInRegistry:
                                warning(
                                    "name conflict between group {0} from repository '{3}' and package {0}:{1} {2}. Ignoring."
                                    .format(name, architecture, version,
                                            repository))
                                break

                            try:
                                groups[group].add(package)
                            except KeyError:
                                groups[group] = Group(group)
                                groups[group].add(package)

                    units.add(package)

    for group in groups.iterkeys():
        units.add(groups[group])

    for virtual in virtuals.iterkeys():
        units.add(virtuals[virtual])

    return units
Beispiel #26
0
 def incr(self, hits=1):
     try:
         self.point.buckets[self.bucket_id()].incr(hits)
     except axisValueError:
         message.warning('increment is not registered')
Beispiel #27
0
 def prologue(self):
     message.warning('no prologue defined')
Beispiel #28
0
 def success(self) :
   'Generic success hook'
   if self.is_success is not None :
     message.warning('success() called after test issue (status : %(status)s)', status=str(self.is_success))
   self.is_success = True
   self.SUCCESS()
Beispiel #29
0
 def prologue(self) :
   message.warning('no prologue defined')
Beispiel #30
0
 def epilogue(self):
     message.warning('no epilogue defined')
Beispiel #31
0
def _install(configuration, installed, package, filepath):
    """ Performs a low-level package installation.

    Parameters
        configuration
            a valid Craft Configuration object.
        installed
            Set having all currently installed units on the system.
        package
            the Package unit to be installed.
        filepath
            absolute filesystem path of the package's archive
            to be installed.
    Raises
        InstallError
            if any error occurs during the installation.
        OSError
            if, in case an operation has failed, it is not possible
            to cleanly recover from it.
    Returns
        True
            if the installation was successfully completed.
    """

    architecture = package.architecture
    name = package.name
    version = package.version
    db = configuration.db()
    package_directory = db + 'installed/' + name + '/' + version + '/' + architecture
    craft_directories = [
        db + 'installed/', db + 'installed/' + name,
        db + 'installed/' + name + '/' + version
    ]

    for each in craft_directories:
        try:
            mkdir(each)
        except OSError:
            pass

    if package in installed:
        message.warning(
            "'{0}' is already installed. Aborting...".format(package))
        raise InstallError(package)

    try:
        mkdir(package_directory)
    except OSError:
        message.warning(
            "failed to create internal directory while installing '{0}'. Aborting..."
            .format(package))
        raise InstallError(package)

    try:
        chdir(package_directory)
    except OSError:
        message.warning(
            "could not access the directory belonging to package '{0}'. Aborting..."
            .format(package))
        raise InstallError(package)

    sha1 = package.has_checksum('sha1')
    if sha1:
        if not filepath:
            message.warning(
                "missing archive filepath for package '{0}'. Aborting...".
                format(package))
            raise InstallError(package)

        if not checksum.sha1(filepath, sha1):
            message.warning(
                "inconsistent archive provided for package '{0}'. Aborting...".
                format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

        package_files = archive.getfiles(filepath)
        if not package_files:
            message.warning(
                "empty archive provided for package '{0}'. Aborting...".format(
                    package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

        try:
            package_files_dump_handle = open('files', 'w')
        except IOError:
            message.warning(
                "could not write the file list for package '{0}'. Aborting...".
                format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)
        else:
            for each in package_files:
                package_files_dump_handle.write(each + '\n')
            package_files_dump_handle.close()

        if not archive.extract(filepath, configuration.root()):
            message.warning(
                "could not extract the archive provided for package '{0}'. Aborting..."
                .format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

    try:
        if not dump.package(package, 'metadata.yml'):
            message.warning(
                "failed to write metadata.yml for package '{0}'. Aborting...".
                format(package))
            raise InstallError(package)
    except IOError:
        raise

    installed.add(package)
    return True
def download(configuration, packages):
    """ Download packages.

    Parameters
        configuration
            a valid Craft Configuration object.
        packages
            an iterable having the Package units to be downloaded.
    Raises
        RepositoryError
            in case an invalid repository was specified.
        DownloadError
            in case of failure.
    Returns
        True
            in case all specified packages have been successfully downloaded.
    """

    grouped_packages = {}
    db = configuration.db()
    repositories = configuration.repositories()
    packages = list(packages)

    for package in list(packages):
        if isfile(db+'/available/'+package.repository+'/cache/'+package.name+'/'+package.version+'/'+package.architecture+'/package.tar.gz'):
            packages.remove(package)

    for package in packages:
        try:
            grouped_packages[package.repository].append(package)
        except KeyError:
            grouped_packages[package.repository] = []
            grouped_packages[package.repository].append(package)

    for repository_name in grouped_packages.iterkeys():
        try:
            repository = repositories[repository_name]
        except KeyError:
            raise RepositoryError(repository_name) 

        try:
            environment.merge(repository['env'])
        except environment.EnvironmentError:
            message.warning("could not merge the environment variables associated to the repository '{0}'!".format(repository_name))
        except KeyError:
            pass

        for package in grouped_packages[repository_name]:
            if package.has_checksum():
                n = package.name
                v = package.version
                a = package.architecture

                directories = [
                    db+'/available',
                    db+'/available/'+repository_name,
                    db+'/available/'+repository_name+'/cache',
                    db+'/available/'+repository_name+'/cache/'+n,
                    db+'/available/'+repository_name+'/cache/'+n+'/'+v,
                    db+'/available/'+repository_name+'/cache/'+n+'/'+v+'/'+a
                ]

                for directory in directories:
                    try:
                        mkdir(directory)
                    except OSError:
                        pass

                try:
                    chdir(db+'/available/'+package.repository+'/cache/'+n+'/'+v+'/'+a)
                except OSError:
                    raise DownloadError(package)

                if not isfile('package.tar.gz'):
                    handler = repository['handler']
                    target = "{0}/{1}/{2}/{3}/package.tar.gz".format(repository['target'], n, v, a)
                    if system(handler+' '+target) != 0:
                        raise DownloadError(package)

        try:
            environment.purge(repository['env'].keys())
        except environment.EnvironmentError:
            message.warning("could not purge the environment variables associated to the repository '{0}'!".format(package.repository))
        except KeyError:
            pass

    return True
Beispiel #33
0
 def set_default_db(cls, **args) :
   message.warning('set default db on mysql')
Beispiel #34
0
 def __exit__(self, exc_type, exc_value, traceback) :
   try :
     self.close()
   except :
     message.warning('cursor close raised exception %(exc)s', exc=sys.exc_info()[1])
def _set(paths):
    """ Loads a Set from one or more YAML files.

    Parameters
        paths
            iterable having the file paths to be loaded.
    Raises
        IOError
            if one of the files could not be read.
        YAMLError
            if one of the files is not a valid YAML file.
        validate.SemanticError
            if one of the files is semantically invalid.
    Returns
        Set containing all units found in the specified files.
    """

    units = Set()
    groups = {}
    virtuals = {}
    registry = Registry()

    for path in paths:
        try:
            definition = yaml(path)
            validate.set(definition)
        except IOError:
            raise
        except YAMLError:
            raise
        except validate.SemanticError:
            raise

        repository = findall('([a-zA-Z0-9]+)', path)[-3]

        for name in definition.iterkeys():
            for version in definition[name].iterkeys():
                for architecture in definition[name][version].iterkeys():
                    data = definition[name][version][architecture]
                    package = Package(name, version, architecture, repository, data)

                    try:
                        registry.add_package(name, version, architecture)
                    except PackageInRegistry:
                        warning("duplicate package found: {0}:{1} {2} from repository '{3}'. Ignoring.".format(name, architecture, version, repository))
                        break
                    except GroupInRegistry:
                        warning("name conflict between group {0} from repository '{3}' and package {0}:{1} {2}. Ignoring.".format(name, architecture, version, repository))
                        break
                    except VirtualPackageInRegistry:
                        warning("name conflict between virtual package {0} from repository '{3}' and package {0}:{1} {2}. Ignoring.".format(name, architecture, version, repository))
                        break

                    if data['provides'] is not None:
                        for virtual in data['provides']:
                            try:
                                registry.add_virtual(virtual)
                            except PackageInRegistry:
                                warning("name conflict between virtual package {0} from repository '{3}' and package {0}:{1} {2}. Ignoring.".format(name, architecture, version, repository))
                                break
                            except GroupInRegistry:
                                warning("name conflict between virtual package {0} from repository '{1}' and group {0}. Ignoring.".format(virtual, repository))
                                break

                            try:
                                virtuals[virtual].provided_by(package)
                            except KeyError:
                                virtuals[virtual] = VirtualPackage(virtual)
                                virtuals[virtual].provided_by(package)

                    if data['groups'] is not None:
                        for group in data['groups']:
                            try:
                                registry.add_group(group)
                            except VirtualPackageInRegistry:
                                warning("name conflict between group {0} from repository '{1}' and virtual package {0}. Ignoring.".format(group, repository))
                                break
                            except PackageInRegistry:
                                warning("name conflict between group {0} from repository '{3}' and package {0}:{1} {2}. Ignoring.".format(name, architecture, version, repository))
                                break

                            try:
                                groups[group].add(package)
                            except KeyError:
                                groups[group] = Group(group)
                                groups[group].add(package)

                    units.add(package)

    for group in groups.iterkeys():
        units.add(groups[group])

    for virtual in virtuals.iterkeys():
        units.add(virtuals[virtual])

    return units
Beispiel #36
0
 def set_default_db(cls, **args):
     message.warning('set default db on mysql')
Beispiel #37
0
def _uninstall(configuration, installed, package, keep_static):
    """ Performs a low-level package uninstallation.

    Parameters
        configuration
            a valid Craft Configuration object.
        installed
            Set having all currently installed units on the system.
        package
            the Package unit to be uninstalled.
        keep_static
            specifies whether the package's static files must be preserved
            or not.
    Raises
        UninstallError
            if any error occurs during the uninstallation.
    Returns
        True
            if the uninstallation was successfully completed.
    """

    architecture = package.architecture
    name = package.name
    version = package.version
    db = configuration.db()
    root = configuration.root()

    if package not in installed:
        message.warning("'{0}' is not installed. Aborting...".format(package))
        raise UninstallError(package)

    try:
        chdir(db + 'installed/' + name + '/' + version + '/' + architecture)
    except OSError:
        message.warning(
            "could not access the directory belonging to package '{0}'.".
            format(package))
        raise UninstallError(package)

    package_files = []
    try:
        handle = open('files')
    except IOError:
        pass
    else:
        package_files = handle.read().splitlines()
        handle.close()

    craft_files = [
        db + 'installed/' + name + '/' + version + '/' + architecture +
        '/metadata.yml', db + 'installed/' + name + '/' + version + '/' +
        architecture + '/files',
        db + 'installed/' + name + '/' + version + '/' + architecture
    ]

    for each in package_files:
        if not access(root + each, W_OK):
            message.warning(
                "cannot remove file '{0}' from package '{1}'.".format(
                    root + each, package))
            raise UninstallError(package)

    for each in craft_files:
        if isfile(each) or isdir(each):
            if not access(each, W_OK):
                message.warning("cannot remove file '{0}'.".format(each))
                raise UninstallError(package)

    if keep_static:
        for each in package.static():
            try:
                message.simple("Attempting to save '{0}' as '{1}'...".format(
                    root + each, root + each + '.craft-old'))
                rename(root + each, root + each + '.craft-old')
            except OSError:
                message.simple(
                    "Could not preserve the following static file: '{0}'.".
                    format(root + each))
                message.simple(
                    "  '{0}' may exist already.".format(root + each +
                                                        '.craft-old'))
                pass

    for each in package_files:
        try:
            if isdir(root + each):
                rmdir(root + each)
            elif isfile(root + each):
                remove(root + each)
        except OSError:
            pass

    for each in craft_files:
        try:
            if isdir(each):
                rmdir(each)
            elif isfile(each):
                remove(each)
        except OSError:
            pass

    try_to_remove = [
        db + 'installed/' + name + '/' + version, db + 'installed/' + name
    ]
    for each in try_to_remove:
        try:
            rmdir(each)
        except OSError:
            break

    installed.remove(package)
    return True
def sync(configuration):
    """ Synchronises enabled repositories from a Craft configuration.

    Parameters
        configuration
            a valid Craft Configuration object.
    Raises
        SyncError
            in case of failure related to the actual synchronisation.
        ClearError
            if the internal clear() call fails and the previously set
            local repository cache is not properly cleared up prior to the
            actual synchronisation.
    Returns
        True
            in case the synchonisation has been successfully executed.
    """

    try:
        clear(configuration, False)
    except ClearError:
        raise

    for name in configuration.repositories().iterkeys():
        repository = configuration.repositories()[name]
        try:
            mkdir(configuration.db()+'available')
        except OSError:
            pass

        try:
            mkdir(configuration.db()+'available/'+name)
        except OSError:
            pass

        try:
            chdir(configuration.db()+'available/'+name)
        except OSError:
            raise SyncError

        try:
            environment.merge(repository['env'])
        except environment.EnvironmentError:
            message.warning("could not merge the environment variables associated to the repository '{0}'!".format(name))
        except KeyError:
            pass

        handler = repository['handler']
        for arch in configuration.architectures():
            target = repository['target']+'/'+arch+'.yml'
            if system(handler+' '+target) != 0:
                message.warning("could not synchronise architecture '{0}' from repository '{1}'!".format(arch, name))

        try:
            environment.purge(repository['env'].keys())
        except environment.EnvironmentError:
            message.warning("could not purge the environment variables associated to the repository '{0}'!".format(name))
        except KeyError:
            pass

    return True
Beispiel #39
0
 def traceback(self, _traceback):
     import traceback
     for details in traceback.format_tb(_traceback):
         for detail in details.strip('\n').split('\n'):
             message.warning(detail)
Beispiel #40
0
 def prologue(self) :
   message.message.verbosity(message.INT_DEBUG)
   message.warning('a warning %(c)d', c=666)
   message.note('a note')
   self.pdb()
 def enqueue(cmd) :
   'just execute here'
   message.debug('enqueue %(cmd)s', cmd=cmd)
   result = subprocess.Popen(cmd.split(' '), env=dict(os.environ, MDB='root='+str(self.mdb.get_root())+',parent='+str(self.mdb.log_id))).wait()
   if result > 0 :
     message.warning('process %(cmd)s returned non zero %(result)d', cmd=cmd, result=result)
Beispiel #42
0
 def ignoring(arg):
     message.warning('Ignoring %(arg)s', arg=arg)
Beispiel #43
0
 def epilogue(self) :
   message.warning('no epilogue defined')
Beispiel #44
0
def sync(configuration):
    """ Synchronises enabled repositories from a Craft configuration.

    Parameters
        configuration
            a valid Craft Configuration object.
    Raises
        SyncError
            in case of failure related to the actual synchronisation.
        ClearError
            if the internal clear() call fails and the previously set
            local repository cache is not properly cleared up prior to the
            actual synchronisation.
    Returns
        True
            in case the synchonisation has been successfully executed.
    """

    try:
        clear(configuration, False)
    except ClearError:
        raise

    for name in configuration.repositories().iterkeys():
        repository = configuration.repositories()[name]
        try:
            mkdir(configuration.db() + 'available')
        except OSError:
            pass

        try:
            mkdir(configuration.db() + 'available/' + name)
        except OSError:
            pass

        try:
            chdir(configuration.db() + 'available/' + name)
        except OSError:
            raise SyncError

        try:
            environment.merge(repository['env'])
        except environment.EnvironmentError:
            message.warning(
                "could not merge the environment variables associated to the repository '{0}'!"
                .format(name))
        except KeyError:
            pass

        handler = repository['handler']
        for arch in configuration.architectures():
            target = repository['target'] + '/' + arch + '.yml'
            if system(handler + ' ' + target) != 0:
                message.warning(
                    "could not synchronise architecture '{0}' from repository '{1}'!"
                    .format(arch, name))

        try:
            environment.purge(repository['env'].keys())
        except environment.EnvironmentError:
            message.warning(
                "could not purge the environment variables associated to the repository '{0}'!"
                .format(name))
        except KeyError:
            pass

    return True
Beispiel #45
0
 def traceback(self, _traceback) :
   import traceback
   for details in traceback.format_tb(_traceback) :
     for detail in details.strip('\n').split('\n') :
       message.warning(detail)
 def prologue(self):
     message.message.verbosity(message.INT_DEBUG)
     ignore()
     message.warning('a warning %(c)d', c=666)
     message.note('a note')
Beispiel #47
0
 def hit(self):
     try:
         self.point.buckets[self.bucket_id()].hit()
     except axisValueError:
         message.warning('hit is not registered')
Beispiel #48
0
def download(configuration, packages):
    """ Download packages.

    Parameters
        configuration
            a valid Craft Configuration object.
        packages
            an iterable having the Package units to be downloaded.
    Raises
        RepositoryError
            in case an invalid repository was specified.
        DownloadError
            in case of failure.
    Returns
        True
            in case all specified packages have been successfully downloaded.
    """

    grouped_packages = {}
    db = configuration.db()
    repositories = configuration.repositories()
    packages = list(packages)

    for package in list(packages):
        if isfile(db + '/available/' + package.repository + '/cache/' +
                  package.name + '/' + package.version + '/' +
                  package.architecture + '/package.tar.gz'):
            packages.remove(package)

    for package in packages:
        try:
            grouped_packages[package.repository].append(package)
        except KeyError:
            grouped_packages[package.repository] = []
            grouped_packages[package.repository].append(package)

    for repository_name in grouped_packages.iterkeys():
        try:
            repository = repositories[repository_name]
        except KeyError:
            raise RepositoryError(repository_name)

        try:
            environment.merge(repository['env'])
        except environment.EnvironmentError:
            message.warning(
                "could not merge the environment variables associated to the repository '{0}'!"
                .format(repository_name))
        except KeyError:
            pass

        for package in grouped_packages[repository_name]:
            if package.has_checksum():
                n = package.name
                v = package.version
                a = package.architecture

                directories = [
                    db + '/available', db + '/available/' + repository_name,
                    db + '/available/' + repository_name + '/cache',
                    db + '/available/' + repository_name + '/cache/' + n, db +
                    '/available/' + repository_name + '/cache/' + n + '/' + v,
                    db + '/available/' + repository_name + '/cache/' + n +
                    '/' + v + '/' + a
                ]

                for directory in directories:
                    try:
                        mkdir(directory)
                    except OSError:
                        pass

                try:
                    chdir(db + '/available/' + package.repository + '/cache/' +
                          n + '/' + v + '/' + a)
                except OSError:
                    raise DownloadError(package)

                if not isfile('package.tar.gz'):
                    handler = repository['handler']
                    target = "{0}/{1}/{2}/{3}/package.tar.gz".format(
                        repository['target'], n, v, a)
                    if system(handler + ' ' + target) != 0:
                        raise DownloadError(package)

        try:
            environment.purge(repository['env'].keys())
        except environment.EnvironmentError:
            message.warning(
                "could not purge the environment variables associated to the repository '{0}'!"
                .format(package.repository))
        except KeyError:
            pass

    return True
Beispiel #49
0
 def __exit__(self, exc_type, exc_value, traceback):
     try:
         self.close()
     except:
         message.warning('cursor close raised exception %(exc)s',
                         exc=sys.exc_info()[1])
Beispiel #50
0
 def ignoring(arg) :
   message.warning('Ignoring %(arg)s', arg=arg)
def _install(configuration, installed, package, filepath):
    """ Performs a low-level package installation.

    Parameters
        configuration
            a valid Craft Configuration object.
        installed
            Set having all currently installed units on the system.
        package
            the Package unit to be installed.
        filepath
            absolute filesystem path of the package's archive
            to be installed.
    Raises
        InstallError
            if any error occurs during the installation.
        OSError
            if, in case an operation has failed, it is not possible
            to cleanly recover from it.
    Returns
        True
            if the installation was successfully completed.
    """

    architecture = package.architecture
    name = package.name
    version = package.version
    db = configuration.db()
    package_directory = db+'installed/'+name+'/'+version+'/'+architecture
    craft_directories = [
        db+'installed/',
        db+'installed/'+name,
        db+'installed/'+name+'/'+version
    ]

    for each in craft_directories:
        try:
            mkdir(each)
        except OSError:
            pass

    if package in installed:
        message.warning("'{0}' is already installed. Aborting...".format(package))
        raise InstallError(package)

    try:
        mkdir(package_directory)
    except OSError:
        message.warning("failed to create internal directory while installing '{0}'. Aborting...".format(package))
        raise InstallError(package)

    try:
        chdir(package_directory)
    except OSError:
        message.warning("could not access the directory belonging to package '{0}'. Aborting...".format(package))
        raise InstallError(package)

    sha1 = package.has_checksum('sha1')
    if sha1:
        if not filepath:
            message.warning("missing archive filepath for package '{0}'. Aborting...".format(package))
            raise InstallError(package)

        if not checksum.sha1(filepath, sha1):
            message.warning("inconsistent archive provided for package '{0}'. Aborting...".format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

        package_files = archive.getfiles(filepath)
        if not package_files:
            message.warning("empty archive provided for package '{0}'. Aborting...".format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

        try:
            package_files_dump_handle = open('files', 'w')
        except IOError:
            message.warning("could not write the file list for package '{0}'. Aborting...".format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)
        else:
            for each in package_files:
                package_files_dump_handle.write(each+'\n')
            package_files_dump_handle.close()

        if not archive.extract(filepath, configuration.root()):
            message.warning("could not extract the archive provided for package '{0}'. Aborting...".format(package))
            try:
                rmtree(package_directory)
            except OSError:
                raise
            raise InstallError(package)

    try:
        if not dump.package(package, 'metadata.yml'):
            message.warning("failed to write metadata.yml for package '{0}'. Aborting...".format(package))
            raise InstallError(package)
    except IOError:
        raise

    installed.add(package)
    return True
def _uninstall(configuration, installed, package, keep_static):
    """ Performs a low-level package uninstallation.

    Parameters
        configuration
            a valid Craft Configuration object.
        installed
            Set having all currently installed units on the system.
        package
            the Package unit to be uninstalled.
        keep_static
            specifies whether the package's static files must be preserved
            or not.
    Raises
        UninstallError
            if any error occurs during the uninstallation.
    Returns
        True
            if the uninstallation was successfully completed.
    """

    architecture = package.architecture
    name = package.name
    version = package.version
    db = configuration.db()
    root = configuration.root()

    if package not in installed:
        message.warning("'{0}' is not installed. Aborting...".format(package))
        raise UninstallError(package)

    try:
        chdir(db+'installed/'+name+'/'+version+'/'+architecture)
    except OSError:
        message.warning("could not access the directory belonging to package '{0}'.".format(package))
        raise UninstallError(package)

    package_files = []
    try:
        handle = open('files')
    except IOError:
        pass
    else:
        package_files = handle.read().splitlines()
        handle.close()

    craft_files = [
        db+'installed/'+name+'/'+version+'/'+architecture+'/metadata.yml',
        db+'installed/'+name+'/'+version+'/'+architecture+'/files',
        db+'installed/'+name+'/'+version+'/'+architecture
    ]

    for each in package_files:
        if not access(root+each, W_OK):
            message.warning("cannot remove file '{0}' from package '{1}'.".format(root+each, package))
            raise UninstallError(package)

    for each in craft_files:
        if isfile(each) or isdir(each):
            if not access(each, W_OK):
                message.warning("cannot remove file '{0}'.".format(each))
                raise UninstallError(package)

    if keep_static:
        for each in package.static():
            try:
                message.simple("Attempting to save '{0}' as '{1}'...".format(root+each, root+each+'.craft-old'))
                rename(root+each, root+each+'.craft-old')
            except OSError:
                message.simple("Could not preserve the following static file: '{0}'.".format(root+each))
                message.simple("  '{0}' may exist already.".format(root+each+'.craft-old'))
                pass

    for each in package_files:
        try:
            if isdir(root+each):
                rmdir(root+each)
            elif isfile(root+each):
                remove(root+each)
        except OSError:
            pass

    for each in craft_files:
        try:
            if isdir(each):
                rmdir(each)
            elif isfile(each):
                remove(each)
        except OSError:
            pass

    try_to_remove = [
        db+'installed/'+name+'/'+version,
        db+'installed/'+name
    ]
    for each in try_to_remove:
        try:
            rmdir(each)
        except OSError:
            break

    installed.remove(package)
    return True
 def __init__(self,
              log_ids=[],
              test_ids=[],
              xml=None,
              threshold=0,
              robust=False,
              previous=None):
     'log_ids is a list of regression roots'
     self.log_ids = log_ids
     s_log_ids = ','.join(map(str, log_ids))
     self.tests = mdb.connection().row_cursor()
     if log_ids:
         # create table of individual runs, but not root node as this may have already summarised coverage
         self.tests.execute(
             'CREATE TEMPORARY TABLE ' + self.invs +
             ' AS SELECT l1.*, goal_id AS master FROM log AS l0 JOIN log AS l1 ON (l0.log_id = l1.root) LEFT OUTER JOIN master ON (l1.log_id = master.log_id) WHERE l1.root IN ('
             + s_log_ids + ');')
         self.tests.execute('SELECT count(*) AS children FROM ' + self.invs)
         children = self.tests.fetchone().children
         if children:
             message.information(
                 '%(log_ids)s %(has)s %(children)d children',
                 log_ids=s_log_ids,
                 children=children,
                 has='have' if len(log_ids) > 1 else 'has')
     # append individual runs as given by test_ids
     if xml:
         xml_ids = xml.xml.xpath('/optimize/test/log_id/text()')
     else:
         xml_ids = []
     if test_ids or xml_ids:
         s_test_ids = ','.join(map(str, test_ids + xml_ids))
         create = ('INSERT INTO ' +
                   self.invs) if log_ids else ('CREATE TEMPORARY TABLE ' +
                                               self.invs + ' AS')
         self.tests.execute(
             create +
             ' SELECT log.*, IFNULL(goal_id, goal.log_id) AS master FROM log LEFT OUTER JOIN master ON (log.log_id = master.log_id) LEFT OUTER JOIN goal ON (log.log_id = goal.log_id) WHERE log.log_id IN ('
             + s_test_ids + ') GROUP BY log_id;')
     self.tests.execute('SELECT count(*) AS tests FROM ' + self.invs)
     tests = self.tests.fetchone().tests
     if tests < 1:
         message.fatal('no tests')
     message.information('starting with %(count)d tests in table %(table)s',
                         count=tests,
                         table=self.invs)
     # check congruency
     self.cvg = mdb.connection().row_cursor()
     rows = self.cvg.execute(
         "SELECT md5_self AS md5, 'md5_self' AS type, invs.master, invs.root FROM point JOIN "
         + self.invs +
         " AS invs ON (invs.master = point.log_id AND point.parent IS NULL) GROUP BY md5;"
     )
     md5 = self.cvg.fetchall()
     if not md5:
         message.fatal('no master')
     elif len(md5) > 1:
         message.fatal('md5 of multiple masters do not match')
     else:
         message.debug('md5 query returns %(rows)d', rows=rows)
     self.master = mdb.accessor(md5=md5[0])
     self.cvg.execute(
         "SELECT DISTINCT(md5_axes) AS md5, 'md5_axes' AS type, invs.master, invs.root FROM point JOIN "
         + self.invs +
         " AS invs ON (invs.master = point.log_id AND point.parent IS NULL) GROUP BY md5;"
     )
     md5 = self.cvg.fetchall()
     if len(md5) > 1:
         message.fatal('md5 of multiple axis masters do not match')
     self.master.axes = md5[0]
     # create status table, collating goal & hits
     self.cvg.execute(
         'CREATE TEMPORARY TABLE ' + self.covg +
         ' (bucket_id INTEGER NOT NULL PRIMARY KEY, goal INTEGER, hits INTEGER, total_hits INTEGER, rhits INTEGER, max_hits INTEGER, tests INTEGER);'
     )
     try:
         self.threshold = float(threshold)
     except:
         self.threshold = 0.0
         message.warning(
             'cannot convert threshold value given "%(arg)s" to float because %(exception)s, using %(threshold)2.1f',
             arg=threshold,
             exception=sys.exc_info()[0],
             threshold=self.threshold)
     self.robust = robust
     self.previous = previous
Beispiel #54
0
 def enqueue(self, cmd) :
   'just execute here'
   message.debug('enqueue %(cmd)s', cmd=cmd)
   result = subprocess.Popen(cmd.split(' '), env=dict(os.environ, MDB='root='+str(mdb_conn.get_root())+',parent='+str(mdb_conn.log_id), PYTHONPATH=os.environ['PYTHONPATH']+':../python')).wait()
   if result > 0 :
     message.warning('process %(cmd)s returned non zero %(result)d', cmd=cmd, result=result)