def project_manager(): form = request.form project_id = form.get('project_id') if project_id: _project = Project.objects.filter(id=project_id).first() else: _project = Project() _project.name = form.get('name') _project.alias = form.get('alias') _project.description = form.get('description') _project.egg = base64.encodestring(request.files['egg'].stream.read()) _project.save() return redirect(url_for('.project'))
def refresh_project(): app = current_app._get_current_object() spiderdir = app.config.get('SPIDERDIR') projects = os.listdir(spiderdir) for name in projects: _project = Project.objects.filter(name=name).first() if not _project: _project = Project() _project.name = name _project.alias = name _project.description = '' _project.save() # spider _spiderpath = os.path.join(spiderdir, name) scrapyexe = app.config.get('SCRAPY_EXE') p = subprocess.Popen([scrapyexe, 'list'], cwd=_spiderpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.wait() stdout = p.stdout.readlines() spiders = [] for spider_name in stdout: spider_name = spider_name.strip() spider = Spider.objects.filter(name=spider_name, project=_project).first() if not spider: spider = Spider() spider.name = spider_name spider.project = _project spider.save() spiders.append(spider) _project.spiders = spiders _project.save() return redirect(url_for('.project'))