def crawl(self, args): """crawl""" if hasattr(args, "name"): items = self.db.getJobs(args.name) else: items = self.db.getJobs() for item in items: logger.info('Starting crawl of: ' + item.name) if hasattr(args, "hash") and args.hash: hashalgorithm = args.hash logger.info('User selected hash-method: ' + args.hash) if not args.hash == item.hashalgorithm: logger.warning('Selected hash-algorithm does not match configured one') else: hashalgorithm = item.hashalgorithm if hasattr(args, 'nometa') and args.nometa == True: metacrawl = False else: metacrawl = True try: self.check(item.name) self.db.lock(item) fs = FS(self.db, item, hashalgorithm, metacrawl) fs.walk() self.db.unlock(item) except CrawlerError: pass except: self.db.session.rollback() item.errors += 1 self.db.session.commit() self.db.unlock(item) raise
def meta(self, args): """update metadata""" if hasattr(args, "name"): items = self.db.getJobs(args.name) else: items = self.db.getJobs() for item in items: logger.info('Getting Metadata of: ' + item.name) try: self.check(item.name) self.db.lock(item) fs = FS(self.db, item) fs.updatemeta() self.db.unlock(item) except CrawlerError: pass except: self.db.session.rollback() item.errors += 1 self.db.session.commit() self.db.unlock(item) raise