Exemple #1
0
    def handle_delete_event(self, event):
        #make sure that all queued uploads are finished before we send a delete event
        #this ensures that all events stay in-order
        if len(self.to_upload) > 0:
            self.upload()

        #if the file doesn't exist and we're a delete event, just drop it
        res = self.database.execute(
            """SELECT * FROM events WHERE localpath=? 
                                    LIMIT 1""", (event.path, ))
        exists = next(res, None)
        if exists is None:
            return

        res = self.database.execute(
            """SELECT * FROM events WHERE localpath=?
                                    AND rev != 0 ORDER BY rev DESC LIMIT 1""",
            (event.path, ))
        latest = next(res, None)
        if latest is not None:
            e = Event(0)
            e.fromseq(latest)
            if e.type & EventType.DELETE:
                #returning because it was already deleted
                return

        #add event to the database
        self.database.execute("INSERT INTO events VALUES (0,?,?,?,?,?,?,?)",
                              event.totuple()[1:])

        self.sender_queue.put(event)
Exemple #2
0
 def enqueue_delete(self, filepath):
     if not path.isdir(filepath):
         e = Event(EventType.DELETE | EventType.LOCAL)
         e.path = get_rel_path(filepath)
         e.time = time() + util.time_diff()
         logging.info("WATCHER "+str(e))
         self.uploader_queue.put(e, True)
Exemple #3
0
 def enqueue_delete(self, filepath):
     if not path.isdir(filepath):
         e = Event(EventType.DELETE | EventType.LOCAL)
         e.path = get_rel_path(filepath)
         e.time = time() + util.time_diff()
         logging.info("WATCHER " + str(e))
         self.uploader_queue.put(e, True)
Exemple #4
0
    def handle_delete_event(self, event):
        #make sure that all queued uploads are finished before we send a delete event
        #this ensures that all events stay in-order
        if len(self.to_upload) > 0:
            self.upload()

        #if the file doesn't exist and we're a delete event, just drop it
        res = self.database.execute("""SELECT * FROM events WHERE localpath=? 
                                    LIMIT 1""", (event.path,))
        exists = next(res, None)
        if exists is None:
            return

        res = self.database.execute("""SELECT * FROM events WHERE localpath=?
                                    AND rev != 0 ORDER BY rev DESC LIMIT 1""", (event.path,))
        latest = next(res, None)
        if latest is not None:
            e = Event(0)
            e.fromseq(latest)
            if e.type & EventType.DELETE:
                #returning because it was already deleted
                return

        #add event to the database
        self.database.execute("INSERT INTO events VALUES (0,?,?,?,?,?,?,?)",
                              event.totuple()[1:])

        self.sender_queue.put(event)
    def __init__(self, train_pwl: PointWithLabelProducer,
                 test_pwl: PointWithLabelProducer, teacher: NetworkTeacher,
                 batch_size: int, learning_rate: float,
                 optimizer: torch.optim.Optimizer, criterion: 'Loss'):
        self.train_pwl = train_pwl
        self.test_pwl = test_pwl
        self.teacher = teacher

        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.optimizer = optimizer
        self.criterion = criterion

        self.setup = Event('setup')
        self.pre_loop = Event('pre_loop')
        self.post_points = Event('post_points')
        self.pre_train = Event('pre_train')
        self.post_train = Event('post_train')
        self.decay_scheduler = Event(
            'decay_scheduler', lambda args, kwargs, result:
            (True, (args[0], args[1], result), kwargs))
        self.decay = Event(
            'decay', lambda args, kwargs, result: (True, (result, ), kwargs))
        self.stopper = Event(
            'stopper', lambda args, kwargs, result: (not result, args, kwargs))
        self.finished = Event('finished')
Exemple #6
0
    def __init__(self, train_ssp: SeqSeqProducer, test_ssp: SeqSeqProducer,
                 teacher: SeqSeqTeacher, batch_size: int, learning_rate: float,
                 optimizers: typing.List[torch.optim.Optimizer],
                 criterion: 'Loss'):
        self.train_ssp = train_ssp
        self.test_ssp = test_ssp
        self.teacher = teacher

        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.optimizers = optimizers
        self.criterion = criterion

        self.setup = Event('setup')
        self.pre_loop = Event('pre_loop')
        self.pre_train = Event('pre_train')
        self.post_train = Event('post_train')
        self.decay_scheduler = Event(
            'decay_scheduler', lambda args, kwargs, result:
            (True, (args[0], args[1], result), kwargs))
        self.decay = Event(
            'decay', lambda args, kwargs, result: (True, (result, ), kwargs))
        self.stopper = Event(
            'stopper', lambda args, kwargs, result: (not result, args, kwargs))
        self.finished = Event('finished')
Exemple #7
0
 def run(self):
     for root, dirs, files in walk(Config().get("core", "syncdir")):
         if self.stopped:
             return
         for file in files:
             filepath = path.join(root, file)
             relpath = path.relpath(filepath, Config().get("core", "syncdir"))
             try:
                 e = Event(EventType.UPDATE | EventType.LOCAL)
                 e.path = relpath
                 e.time = path.getmtime(filepath) + util.time_diff()
                 self.uploader_queue.put(e, True)
             except OSError:
                 logging.warning("Couldn't get a modification time for "
                                 +filepath+". Ignoring file")
Exemple #8
0
    def handle_update_event(self, event):
        filepath = os.path.join(Config().get("core", "syncdir"), event.path)

        #first, copy the file over to a temporary directory, get its hash,
        #upload it, and then move it to the filename with that hash value
        handle, tmppath = mkstemp(dir=Config().get("core", "cachedir"))
        os.close(handle) #we don't really want it open, we just want a good name
        try:
            copy2(filepath, tmppath)
        except IOError:
            logging.warning("Dropping update event because file was deleted before we could upload it: %s" % (str(event)))
            return

        #get the mode of the file
        stats = os.stat(filepath)
        event.permissions = str(stat.S_IMODE(stats.st_mode))

        #hash the temporary file
        event.hash = hash(tmppath)[0]
        logging.debug("HASHED  "+str(event))

        #make sure the most recent version of this file doesn't match this one
        #otherwise it's pointless to re-upload it
        res = self.database.execute("""SELECT * FROM events WHERE localpath=?
                                    AND rev != 0 ORDER BY rev DESC LIMIT 1""", (event.path,))
        latest = next(res, None)
        if latest is not None:
            e = Event(0)
            e.fromseq(latest)
            if e.hash == event.hash:
                #returning because hashes are equal
                #but first, remove the temporary file in the cache
                os.remove(tmppath)
                return

        res = self.database.execute("SELECT * FROM events WHERE hash=? AND rev!=0",
                                   (event.hash,))
        sameHash = next(res, None)
        #if this file isn't already uploaded, add it to the list to upload, and
        #upload them in a batch when we have a chance
        if sameHash is None:
            self.to_upload.append((event, tmppath))
            return

        e = Event(0)
        e.fromseq(sameHash)
        event.storagekey = e.storagekey

        #add event to the database
        self.database.execute("INSERT INTO events VALUES (0,?,?,?,?,?,?,?)",
                              event.totuple()[1:])

        #move tmp file to hash-named file in cache directory
        cachepath = os.path.join(Config().get("core", "cachedir"), event.hash)
        move(tmppath, cachepath)

        self.sender_queue.put(event)
Exemple #9
0
    def handle_update_event(self, event):
        filepath = os.path.join(Config().get("core", "syncdir"), event.path)

        #first, copy the file over to a temporary directory, get its hash,
        #upload it, and then move it to the filename with that hash value
        handle, tmppath = mkstemp(dir=Config().get("core", "cachedir"))
        os.close(
            handle)  #we don't really want it open, we just want a good name
        try:
            copy2(filepath, tmppath)
        except IOError:
            logging.warning(
                "Dropping update event because file was deleted before we could upload it: %s"
                % (str(event)))
            return

        #get the mode of the file
        stats = os.stat(filepath)
        event.permissions = str(stat.S_IMODE(stats.st_mode))

        #hash the temporary file
        event.hash = hash(tmppath)[0]
        logging.debug("HASHED  " + str(event))

        #make sure the most recent version of this file doesn't match this one
        #otherwise it's pointless to re-upload it
        res = self.database.execute(
            """SELECT * FROM events WHERE localpath=?
                                    AND rev != 0 ORDER BY rev DESC LIMIT 1""",
            (event.path, ))
        latest = next(res, None)
        if latest is not None:
            e = Event(0)
            e.fromseq(latest)
            if e.hash == event.hash:
                #returning because hashes are equal
                #but first, remove the temporary file in the cache
                os.remove(tmppath)
                return

        res = self.database.execute(
            "SELECT * FROM events WHERE hash=? AND rev!=0", (event.hash, ))
        sameHash = next(res, None)
        #if this file isn't already uploaded, add it to the list to upload, and
        #upload them in a batch when we have a chance
        if sameHash is None:
            self.to_upload.append((event, tmppath))
            return

        e = Event(0)
        e.fromseq(sameHash)
        event.storagekey = e.storagekey

        #add event to the database
        self.database.execute("INSERT INTO events VALUES (0,?,?,?,?,?,?,?)",
                              event.totuple()[1:])

        #move tmp file to hash-named file in cache directory
        cachepath = os.path.join(Config().get("core", "cachedir"), event.hash)
        move(tmppath, cachepath)

        self.sender_queue.put(event)