Beispiel #1
0
  def testVisitPath(self):
    f.makedirs('TestDir/a/b/c/d/e/f')
    fi = open('TestDir/a/b/c/d/D0','w')
    fi.write("hi\n")
    fi.close
    seen = set()
    def collector(x):
      seen.add(x)
    top = 'TestDir/a'
    last = 'TestDir/a/b/c/d'
    absTop = os.path.normpath(top)
    expected = set([absTop])
    for i in [['b'],['b','c'],['b','c','d']]:
      expected.add(os.path.join(absTop,os.sep.join(i)))
    f.visitPath(top,last,collector)
    assert expected == seen, 'but x-s=%s and s-x=%s'%(expected-seen,seen-expected)

    seen.clear()
    top = 'TestDir/a/b'
    last = 'TestDir/a/b/c/d/D0'
    normTop = os.path.normpath(top)
    expected = set([normTop])
    for i in [['c'],['c','d']]:
      expected.add(os.path.join(normTop,os.sep.join(i)))
    f.visitPath(top,last,collector)
    assert expected == seen, 'but x-s=%s and s-x=%s'%(expected-seen,seen-expected)

    #Test for non-existent leaf
    assert_raises(OSError,f.visitPath,'TestDir','TestDir/A/BB',collector)

    #Test for rootDir not abover fullPath
    assert_raises(OSError,f.visitPath,'TestDir/A/B','TestDir/A',collector)
  def testNewEntryPermissions(self):
    dirPermissions=0707
    dumpPermissions=0500
    sfl = SilentFakeLogger()
    j = JDS.JsonDumpStorage(root=self.testDir,dirPermissions=dirPermissions,dumpPermissions=dumpPermissions,logger=sfl)
    u = str(socorro_uuid.uuid1())
    f1, f2 = j.newEntry(u)
    f1.close()
    f2.close()
    jpath = j.getJson(u)
    gotPermissions = stat.S_IMODE(os.stat(jpath)[0])
    assert stat.S_IMODE(os.stat(jpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (jpath, dumpPermissions, gotPermissions)

    dpath = j.getDump(u)
    gotPermissions = stat.S_IMODE(os.stat(dpath)[0])
    assert stat.S_IMODE(os.stat(dpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (dpath, dumpPermissions, gotPermissions)

    udir = os.path.split(dpath)[0]
    datePath = os.path.abspath(os.path.join(udir,os.readlink(os.path.splitext(dpath)[0])))
    namePath = os.path.abspath(os.path.splitext(dpath)[0])
    topPath = os.path.abspath(self.testDir)
    dailies = os.listdir(topPath)

    def assertPermVisitor(p):
      gotPerm = stat.S_IMODE(os.stat(p)[0])
      assert dirPermissions == gotPerm, "%s: Expected %0o, got %0o"%(p,dirPermissions,gotPerm)
    for d in dailies:
      # visitPath quietly ignores a file as the leaf
      socorro_fs.visitPath(os.path.join(topPath,d),datePath,assertPermVisitor)
      socorro_fs.visitPath(os.path.join(topPath,d),namePath,assertPermVisitor)
  def testNewEntryPermissions(self):
    dirPermissions=0707
    dumpPermissions=0500
    sfl = SilentFakeLogger()
    j = JDS.JsonDumpStorage(root=self.testDir,dirPermissions=dirPermissions,dumpPermissions=dumpPermissions,logger=sfl)
    u = str(socorro_uuid.uuid1())
    f1, f2 = j.newEntry(u)
    f1.close()
    f2.close()
    jpath = j.getJson(u)
    gotPermissions = stat.S_IMODE(os.stat(jpath)[0])
    assert stat.S_IMODE(os.stat(jpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (jpath, dumpPermissions, gotPermissions)

    dpath = j.getDump(u)
    gotPermissions = stat.S_IMODE(os.stat(dpath)[0])
    assert stat.S_IMODE(os.stat(dpath)[0]) == dumpPermissions, "%s: Expected %o, got %o" % (dpath, dumpPermissions, gotPermissions)

    udir = os.path.split(dpath)[0]
    datePath = os.path.abspath(os.path.join(udir,os.readlink(os.path.splitext(dpath)[0])))
    namePath = os.path.abspath(os.path.splitext(dpath)[0])
    topPath = os.path.abspath(self.testDir)
    dailies = os.listdir(topPath)

    def assertPermVisitor(p):
      gotPerm = stat.S_IMODE(os.stat(p)[0])
      assert dirPermissions == gotPerm, "%s: Expected %0o, got %0o"%(p,dirPermissions,gotPerm)
    for d in dailies:
      # visitPath quietly ignores a file as the leaf
      socorro_fs.visitPath(os.path.join(topPath,d),datePath,assertPermVisitor)
      socorro_fs.visitPath(os.path.join(topPath,d),namePath,assertPermVisitor)
Beispiel #4
0
  def testVisitPath(self):
    f.makedirs('TestDir/a/b/c/d/e/f')
    fi = open('TestDir/a/b/c/d/D0','w')
    fi.write("hi\n")
    fi.close
    seen = set()
    def collector(x):
      seen.add(x)
    top = 'TestDir/a'
    last = 'TestDir/a/b/c/d'
    absTop = os.path.normpath(top)
    expected = set([absTop])
    for i in [['b'],['b','c'],['b','c','d']]:
      expected.add(os.path.join(absTop,os.sep.join(i)))
    f.visitPath(top,last,collector)
    assert expected == seen, 'but x-s=%s and s-x=%s'%(expected-seen,seen-expected)

    seen.clear()
    top = 'TestDir/a/b'
    last = 'TestDir/a/b/c/d/D0'
    normTop = os.path.normpath(top)
    expected = set([normTop])
    for i in [['c'],['c','d']]:
      expected.add(os.path.join(normTop,os.sep.join(i)))
    f.visitPath(top,last,collector)
    assert expected == seen, 'but x-s=%s and s-x=%s'%(expected-seen,seen-expected)

    #Test for non-existent leaf
    assert_raises(OSError,f.visitPath,'TestDir','TestDir/A/BB',collector)

    #Test for rootDir not abover fullPath
    assert_raises(OSError,f.visitPath,'TestDir/A/B','TestDir/A',collector)
Beispiel #5
0
 def makeNameDir(self, ooid, timestamp=None):
     """Make sure the name directory exists, and return its path, and list
     of path components. Raises OSError on failure"""
     npath, nparts = self.namePath(ooid, timestamp)
     #self.logger.debug(
     #  "%s - trying makedirs %s",
     #  threading.currentThread().getName(),
     #  npath
     #)
     um = self.osModule.umask(0)
     try:
         try:
             socorro_fs.makedirs(npath, self.dirPermissions, self.osModule)
         except OSError:
             if not self.osModule.path.isdir(npath):
                 raise
     finally:
         self.osModule.umask(um)
     if self.dumpGID:
         socorro_fs.visitPath(
           os.path.join(*nparts[:2]),
           npath,
           self.chownGidVisitor
         )
     return npath, nparts
Beispiel #6
0
 def makeDateDir(self, date, webheadName=None):
     """Assure existence of date directory for the given date, return path,
     and list of components"""
     dpath, dparts = self.datePath(date, webheadName)
     um = self.osModule.umask(0)
     try:
         try:
             socorro_fs.makedirs(dpath, self.dirPermissions, self.osModule)
         except OSError, e:
             if not self.osModule.path.isdir(dpath):
                 #self.logger.debug(
                 #  "%s - in makeDateDir, got not isdir(%s): %s",
                 #  threading.currentThread().getName(),
                 #  dpath,
                 #  e
                 #)
                 raise
     finally:
         self.osModule.umask(um)
     if self.dumpGID:
         socorro_fs.visitPath(
           os.path.join(*dparts[:2]),
           dpath,
           self.chownGidVisitor
         )
     return dpath, dparts
Beispiel #7
0
 def makeDateDir(self, date, webheadName=None):
     """Assure existence of date directory for the given date, return path,
     and list of components"""
     dpath, dparts = self.datePath(date, webheadName)
     um = self.osModule.umask(0)
     try:
         try:
             socorro_fs.makedirs(dpath, self.dirPermissions, self.osModule)
         except OSError:
             if not self.osModule.path.isdir(dpath):
                 raise
     finally:
         self.osModule.umask(um)
     if self.dumpGID:
         socorro_fs.visitPath(
           os.path.join(*dparts[:2]),
           dpath,
           self.chownGidVisitor
         )
     return dpath, dparts
Beispiel #8
0
 def makeDateDir(self, date, webheadName=None):
     """Assure existence of date directory for the given date, return path,
     and list of components"""
     dpath, dparts = self.datePath(date, webheadName)
     um = self.osModule.umask(0)
     try:
         try:
             socorro_fs.makedirs(dpath, self.dirPermissions, self.osModule)
         except OSError, e:
             if not self.osModule.path.isdir(dpath):
                 #self.logger.debug(
                 #  "%s - in makeDateDir, got not isdir(%s): %s",
                 #  threading.currentThread().getName(),
                 #  dpath,
                 #  e
                 #)
                 raise
     finally:
         self.osModule.umask(um)
     if self.dumpGID:
         socorro_fs.visitPath(os.path.join(*dparts[:2]), dpath,
                              self.chownGidVisitor)
     return dpath, dparts
class JsonDumpStorage(socorro_dumpStorage.DumpStorage):
    """
    This class implements a file system storage scheme for the JSON and dump
    files of the Socorro project. It create a tree with two branches: the name
    branch and the date branch.
     - The name branch consists of paths based on the first 8 characters of the
       crash_id file name. It holds the two data files and a relative symbolic
       link to the date branch directory associated with the particular
       crash_id.  see socorro.lib.ooid.py for details of date and depth
       encoding within the crash_id
       For the crash_id:  22adfb61-f75b-11dc-b6be-001322081225
        - the json file is stored as
          %(root)s/%(daypart)s/name/22/ad/22adfb61-f75b-11dc-b6be-001322081225
          .json
        - the dump file is stored as
          %(root)s/name/22/ad/22adfb61-f75b-11dc-b6be-001322081225.dump
        - the symbolic link is stored as
          %(root)s/name/22/ad/22adfb61-f75b-11dc-b6be-001322081225
          and (see below) references
          %(toDateFromName)s/date/2008/12/25/12/05/webhead01_0
     - The date branch consists of paths based on the year, month, day, hour,
       minute-segment, webhead host name and a small sequence number.
       For each crash_id, it holds a relative symbolic link referring to the
       actual storage (name) directory holding the data for that crash_id.
       For the crash_id above, submitted at 2008-12-25T12:05 from webhead01
        - the symbolic link is stored as
          %(root)s/date/2008/09/30/12/05/webhead01_0/22adfb61-f75b-11dc-b6be-
          001322081225 and references %(toNameFromDate)s/name/22/ad/

    Note: The symbolic links are relative, so they begin with some rounds of
    '../'. This is to avoid issues that might arise from variously mounted nfs
    volumes. If the layout changes, self.toNameFromDate and toDateFromName
    must be changed to match, as well as a number of the private methods.

    Note: If so configured, the bottom nodes in the date path will be
    %(webheadName)s_n for n in range(N) for some reasonable (5, perhaps) N.
    Files are placed into these buckets in rotation.
    """

    #--------------------------------------------------------------------------
    def __init__(self, root=".", osModule=os, **kwargs):
        """
        Take note of our root directory and other necessities.
        Yes, it is perfectly legal to call super(...).__init__() after doing
        some other code. As long as you expect the behavior you get, anyway...
        """
        kwargs.setdefault('minutesPerSlot', 1)
        kwargs.setdefault('subSlotCount', 1)  # that is: use xxx_0 every time
        # by default
        super(JsonDumpStorage, self).__init__(root=root,
                                              osModule=osModule,
                                              **kwargs)
        tmp = kwargs.get('cleanIndexDirectories', 'false')
        self.cleanIndexDirectories = 'true' == tmp.lower()
        self.jsonSuffix = kwargs.get('jsonSuffix', '.json')
        if not self.jsonSuffix.startswith('.'):
            self.jsonSuffix = ".%s" % (self.jsonSuffix)
        self.dumpSuffix = kwargs.get('dumpSuffix', '.dump')
        if not self.dumpSuffix.startswith('.'):
            self.dumpSuffix = ".%s" % (self.dumpSuffix)
        self.logger = kwargs.get('logger', socorro_util.FakeLogger())

    #--------------------------------------------------------------------------
    def new_entry(self,
                  crash_id,
                  raw_crash,
                  dumps_dict,
                  webhead_host_name='webhead01',
                  timestamp=None):
        if not isinstance(dumps_dict, collections.Mapping):
            dumps_dict = {self.dump_field: dumps_dict}

        name_dir, date_dir = super(JsonDumpStorage, self).newEntry(
            crash_id,
            datetimeFromISOdateString(raw_crash['submitted_timestamp']),
            webhead_host_name)

        raw_crash_pathname = os.path.join(name_dir, crash_id + self.jsonSuffix)
        with open(raw_crash_pathname, "w") as rcf:
            json.dump(raw_crash, rcf)

        for dump_name, dump in dumps_dict.iteritems():
            full_dump_name = self.dump_file_name(crash_id, dump_name)
            dump_pathname = os.path.join(name_dir, full_dump_name)
            with open(dump_pathname, "w") as dp:
                dp.write(dump)
            self.osModule.chmod(dump_pathname, self.dumpPermissions)

        name_depth = socorro_ooid.depthFromOoid(crash_id)
        if not name_depth:
            name_depth = 4
        rparts = [
            os.path.pardir,
        ] * (1 + name_depth)
        rparts.append(self.dateName)
        date_depth = 2  # .../hh/mm_slot...
        if webhead_host_name and self.subSlotCount:
            date_depth = 3  # .../webHeadName_slot
        date_parts = date_dir.split(os.path.sep)[-date_depth:]
        rparts.extend(date_parts)
        self.osModule.symlink(os.path.sep.join(rparts),
                              os.path.join(name_dir, crash_id))

    #--------------------------------------------------------------------------
    def newEntry(self, crash_id, webheadHostName='webhead01', timestamp=None):
        """
        Sets up the name and date storage directory branches for the given
        crash_id. Creates any directories that it needs along the path to the
        appropriate storage location.
        Creates two relative symbolic links: the date branch link pointing to
        the name directory holding the files;
        the name branch link pointing to the date branch directory holding that
        link.
        Returns a 2-tuple containing files open for writing: (jsonfile,
        dumpfile)
        If self.dumpGID, then the file tree from root to and including the data
        files are chown'd
        If self.dumpPermissions, then chmod is called on the data files
        """
        # note: after this call, dateDir already holds link to nameDir
        nameDir, dateDir = super(JsonDumpStorage,
                                 self).newEntry(crash_id, timestamp,
                                                webheadHostName)
        df, jf = None, None
        jname = os.path.join(nameDir, crash_id + self.jsonSuffix)
        try:
            jf = open(jname, 'w')
        except IOError, x:
            if 2 == x.errno:
                nameDir = self.makeNameDir(crash_id, timestamp)  # deliberately
                # leave this dir behind if next line throws
                jf = open(jname, 'w')
            else:
                raise x
        try:
            # Do all this in a try/finally block to unroll in case of error
            self.osModule.chmod(jname, self.dumpPermissions)
            dname = os.path.join(nameDir, crash_id + self.dumpSuffix)
            df = open(dname, 'w')
            self.osModule.chmod(dname, self.dumpPermissions)
            nameDepth = socorro_ooid.depthFromOoid(crash_id)
            if not nameDepth:
                nameDepth = 4
            rparts = [
                os.path.pardir,
            ] * (1 + nameDepth)
            rparts.append(self.dateName)
            dateDepth = 2  # .../hh/mm_slot...
            if webheadHostName and self.subSlotCount:
                dateDepth = 3  # .../webHeadName_slot
            dateParts = dateDir.split(os.path.sep)[-dateDepth:]
            rparts.extend(dateParts)
            self.osModule.symlink(os.path.sep.join(rparts),
                                  os.path.join(nameDir, crash_id))
            if self.dumpGID:

                def chown1(path):
                    self.osModule.chown(path, -1, self.dumpGID)

                socorro_fs.visitPath(
                    self.root, os.path.join(nameDir,
                                            crash_id + self.jsonSuffix),
                    chown1, self.osModule)
                self.osModule.chown(
                    os.path.join(nameDir, crash_id + self.dumpSuffix), -1,
                    self.dumpGID)
                # socorro_fs.visitPath(self.root,
                #   os.path.join(dateDir,crash_id),
                #   chown1
                # )
        finally:
            if not jf or not df:
                if jf:
                    jf.close()
                if df:
                    df.close()
                try:
                    self.osModule.unlink(os.path.join(dateDir, crash_id))
                except Exception:
                    pass  # ok if not there
                try:
                    self.osModule.unlink(os.path.join(nameDir, crash_id))
                except Exception:
                    pass  # ok if not there
                df, jf = None, None
        return (jf, df)