def project_remove(request, project): if request.method == 'POST': path = merge(os.path.abspath(os.getcwd()), PROJECTS_FOLDER) if project: project_path = merge(path, project) shutil.rmtree(project_path) return HttpResponse(json.dumps('1'))
def build_egg(project): work_path = os.getcwd() try: path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) print('Path', path) project_path = merge(path, project) os.chdir(project_path) settings = config(project_path, 'settings', 'default') print('Settings', settings) create_default_setup_py(project_path, settings=settings, project=project) d = tempfile.mkdtemp(prefix="gerapy-") o = open(os.path.join(d, "stdout"), "wb") e = open(os.path.join(d, "stderr"), "wb") retry_on_eintr( check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e) o.close() e.close() egg = glob.glob(os.path.join(d, '*.egg'))[0] # 删除原文件 if find_egg(project_path): os.remove(merge(project_path, find_egg(project_path))) shutil.move(egg, project_path) return merge(project_path, find_egg(project_path)) except Exception as e: print(e.args) finally: os.chdir(work_path)
def project_file_rename(request): if request.method == 'POST': data = json.loads(request.body) pre = merge(data['path'], data['pre']) new = merge(data['path'], data['new']) os.rename(pre, new) return HttpResponse('1')
def project_index(request): if request.method == 'GET': path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) files = os.listdir(path) project_list = [] for file in files: if os.path.isdir(merge(path, file)) and not file in IGNORES: project_list.append({'name': file}) return HttpResponse(json.dumps(project_list))
def project_build(request, project): print(request.body) path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) project_path = merge(path, project) if request.method == 'GET': print(project) egg = find_egg(project_path) print('Egg', egg) if egg: built_at = os.path.getmtime(merge(project_path, egg)) print(built_at) if not Project.objects.filter(name=project): Project(name=project, built_at=built_at, egg=egg).save() model = Project.objects.get(name=project) else: model = Project.objects.get(name=project) model.built_at = built_at model.egg = egg model.save() print(model) dict = model_to_dict(model) print(dict) del dict['clients'] localtime = time.localtime(int(built_at)) dict['built_at'] = time.strftime('%Y-%m-%d %H:%M:%S', localtime) return HttpResponse(json.dumps(dict)) else: return HttpResponse(json.dumps({'name': project})) elif request.method == 'POST': data = json.loads(request.body) print(data) description = data['description'] print(description) build_project(project) egg = find_egg(project_path) built_at = time.time() if not Project.objects.filter(name=project): Project(name=project, description=description, built_at=built_at, egg=egg).save() model = Project.objects.get(name=project) else: model = Project.objects.get(name=project) model.built_at = built_at model.egg = egg model.description = description model.save() print(model) dict = model_to_dict(model) print(dict) del dict['clients'] localtime = time.localtime(int(built_at)) dict['built_at'] = time.strftime('%Y-%m-%d %H:%M:%S', localtime) return HttpResponse(json.dumps(dict))
def init(folder): if not folder: folder = 'gerapy' # 执行路径 execute_path = os.getcwd() folder_path = merge(execute_path, folder) # 创建folder,默认gerapy if not os.path.exists(folder_path): os.mkdir(folder_path) # 创建projects文件夹,存放scrapy项目 os.chdir(folder_path) projects_folder = merge(folder_path, PROJECTS_FOLDER) if not os.path.exists(projects_folder): os.mkdir(projects_folder)
def project_deploy(request, id, project): if request.method == 'GET': path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) project_path = merge(path, project) egg = find_egg(project_path) egg_file = open(merge(project_path, egg), 'rb') deploy_version = time.time() client_model = Client.objects.get(id=id) project_model = Project.objects.get(name=project) Deploy.objects.filter(client=client_model, project=project_model).delete() deploy = Deploy.objects.update_or_create(client=client_model, project=project_model, description=project_model.description) scrapyd = ScrapydAPI(scrapyd_url(client_model.ip, client_model.port)) result = scrapyd.add_version(project, int(deploy_version), egg_file.read()) return HttpResponse(result)
def create_default_setup_py(path, **kwargs): with open(merge(path, 'setup.py'), 'w') as f: print(kwargs) file = _SETUP_PY_TEMPLATE % kwargs print(file) f.write(file) f.close()
def config(path, section, option, name='scrapy.cfg', default=None): try: cf = configparser.ConfigParser() cfg_path = merge(path, name) cf.read(cfg_path) return cf.get(section, option) except configparser.NoOptionError: return default
def project_file_update(request): if request.method == 'POST': data = json.loads(request.body) path = merge(data['path'], data['label']) code = data['code'] with open(path, 'w') as f: f.write(code) return HttpResponse(json.dumps('1'))
def project_create(request): if request.method == 'POST': data = json.loads(request.body) data['configurable'] = 1 path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) cmd = 'cd ' + path + '&& scrapy startproject ' + data.get('name') result = os.system(cmd) if result == 0: Project.objects.update_or_create(**data) return HttpResponse(data.get('name'))
def index_status(request): if request.method == 'GET': clients = Client.objects.all() data = { 'success': 0, 'error': 0, 'project': 0, } for client in clients: try: requests.get(scrapyd_url(client.ip, client.port), timeout=1) data['success'] += 1 except ConnectionError: data['error'] += 1 path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) files = os.listdir(path) for file in files: if os.path.isdir(merge(path, file)) and not file in IGNORES: data['project'] += 1 return HttpResponse(json.dumps(data))
def project_file_delete(request): if request.method == 'POST': data = json.loads(request.body) path = merge(data['path'], data['label']) result = os.remove(path) return HttpResponse(json.dumps(result))
def project_file_create(request): if request.method == 'POST': data = json.loads(request.body) path = merge(data['path'], data['name']) open(path, 'w').close() return HttpResponse('1')
def project_file(request): if request.method == 'POST': data = json.loads(request.body) path = merge(data['path'], data['label']) with open(path, 'r') as f: return HttpResponse(f.read())
def project_tree(request, name): if request.method == 'GET': path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER)) tree = get_tree(merge(path, name)) return HttpResponse(json.dumps(tree))