def crawl(self, args):
"""crawl"""
if hasattr(args, "name"):
items = self.db.getJobs(args.name)
else:
items = self.db.getJobs()
for item in items:
logger.info('Starting crawl of: ' + item.name)
if hasattr(args, "hash") and args.hash:
hashalgorithm = args.hash
logger.info('User selected hash-method: ' + args.hash)
if not args.hash == item.hashalgorithm:
logger.warning('Selected hash-algorithm does not match configured one')
else:
hashalgorithm = item.hashalgorithm
if hasattr(args, 'nometa') and args.nometa == True:
metacrawl = False
else:
metacrawl = True
try:
self.check(item.name)
self.db.lock(item)
fs = FS(self.db, item, hashalgorithm, metacrawl)
fs.walk()
self.db.unlock(item)
except CrawlerError:
pass
except:
self.db.session.rollback()
item.errors += 1
self.db.session.commit()
self.db.unlock(item)
raise
def meta(self, args):
"""update metadata"""
if hasattr(args, "name"):
items = self.db.getJobs(args.name)
else:
items = self.db.getJobs()
for item in items:
logger.info('Getting Metadata of: ' + item.name)
try:
self.check(item.name)
self.db.lock(item)
fs = FS(self.db, item)
fs.updatemeta()
self.db.unlock(item)
except CrawlerError:
pass
except:
self.db.session.rollback()
item.errors += 1
self.db.session.commit()
self.db.unlock(item)
raise
