def refresh_project():
app = current_app._get_current_object()
spiderdir = app.config.get('SPIDERDIR')
projects = os.listdir(spiderdir)
for name in projects:
_project = Project.objects.filter(name=name).first()
if not _project:
_project = Project()
_project.name = name
_project.alias = name
_project.description = ''
_project.save()
# spider
_spiderpath = os.path.join(spiderdir, name)
scrapyexe = app.config.get('SCRAPY_EXE')
p = subprocess.Popen([scrapyexe, 'list'],
cwd=_spiderpath,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.wait()
stdout = p.stdout.readlines()
spiders = []
for spider_name in stdout:
spider_name = spider_name.strip()
spider = Spider.objects.filter(name=spider_name,
project=_project).first()
if not spider:
spider = Spider()
spider.name = spider_name
spider.project = _project
spider.save()
spiders.append(spider)
_project.spiders = spiders
_project.save()
return redirect(url_for('.project'))
