def project_remove(request, project):
if request.method == 'POST':
path = merge(os.path.abspath(os.getcwd()), PROJECTS_FOLDER)
if project:
project_path = merge(path, project)
shutil.rmtree(project_path)
return HttpResponse(json.dumps('1'))
def build_egg(project):
work_path = os.getcwd()
try:
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
print('Path', path)
project_path = merge(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
print('Settings', settings)
create_default_setup_py(project_path,
settings=settings,
project=project)
d = tempfile.mkdtemp(prefix="gerapy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
# 删除原文件
if find_egg(project_path):
os.remove(merge(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return merge(project_path, find_egg(project_path))
except Exception as e:
print(e.args)
finally:
os.chdir(work_path)
def project_file_rename(request):
if request.method == 'POST':
data = json.loads(request.body)
pre = merge(data['path'], data['pre'])
new = merge(data['path'], data['new'])
os.rename(pre, new)
return HttpResponse('1')
def project_index(request):
if request.method == 'GET':
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
files = os.listdir(path)
project_list = []
for file in files:
if os.path.isdir(merge(path, file)) and not file in IGNORES:
project_list.append({'name': file})
return HttpResponse(json.dumps(project_list))
def project_build(request, project):
print(request.body)
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
project_path = merge(path, project)
if request.method == 'GET':
print(project)
egg = find_egg(project_path)
print('Egg', egg)
if egg:
built_at = os.path.getmtime(merge(project_path, egg))
print(built_at)
if not Project.objects.filter(name=project):
Project(name=project, built_at=built_at, egg=egg).save()
model = Project.objects.get(name=project)
else:
model = Project.objects.get(name=project)
model.built_at = built_at
model.egg = egg
model.save()
print(model)
dict = model_to_dict(model)
print(dict)
del dict['clients']
localtime = time.localtime(int(built_at))
dict['built_at'] = time.strftime('%Y-%m-%d %H:%M:%S', localtime)
return HttpResponse(json.dumps(dict))
else:
return HttpResponse(json.dumps({'name': project}))
elif request.method == 'POST':
data = json.loads(request.body)
print(data)
description = data['description']
print(description)
build_project(project)
egg = find_egg(project_path)
built_at = time.time()
if not Project.objects.filter(name=project):
Project(name=project, description=description, built_at=built_at, egg=egg).save()
model = Project.objects.get(name=project)
else:
model = Project.objects.get(name=project)
model.built_at = built_at
model.egg = egg
model.description = description
model.save()
print(model)
dict = model_to_dict(model)
print(dict)
del dict['clients']
localtime = time.localtime(int(built_at))
dict['built_at'] = time.strftime('%Y-%m-%d %H:%M:%S', localtime)
return HttpResponse(json.dumps(dict))
def init(folder):
if not folder:
folder = 'gerapy'
# 执行路径
execute_path = os.getcwd()
folder_path = merge(execute_path, folder)
# 创建folder，默认gerapy
if not os.path.exists(folder_path):
os.mkdir(folder_path)
# 创建projects文件夹，存放scrapy项目
os.chdir(folder_path)
projects_folder = merge(folder_path, PROJECTS_FOLDER)
if not os.path.exists(projects_folder):
os.mkdir(projects_folder)
def project_deploy(request, id, project):
if request.method == 'GET':
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
project_path = merge(path, project)
egg = find_egg(project_path)
egg_file = open(merge(project_path, egg), 'rb')
deploy_version = time.time()
client_model = Client.objects.get(id=id)
project_model = Project.objects.get(name=project)
Deploy.objects.filter(client=client_model, project=project_model).delete()
deploy = Deploy.objects.update_or_create(client=client_model, project=project_model,
description=project_model.description)
scrapyd = ScrapydAPI(scrapyd_url(client_model.ip, client_model.port))
result = scrapyd.add_version(project, int(deploy_version), egg_file.read())
return HttpResponse(result)
def create_default_setup_py(path, **kwargs):
with open(merge(path, 'setup.py'), 'w') as f:
print(kwargs)
file = _SETUP_PY_TEMPLATE % kwargs
print(file)
f.write(file)
f.close()
def config(path, section, option, name='scrapy.cfg', default=None):
try:
cf = configparser.ConfigParser()
cfg_path = merge(path, name)
cf.read(cfg_path)
return cf.get(section, option)
except configparser.NoOptionError:
return default
def project_file_update(request):
if request.method == 'POST':
data = json.loads(request.body)
path = merge(data['path'], data['label'])
code = data['code']
with open(path, 'w') as f:
f.write(code)
return HttpResponse(json.dumps('1'))
def project_create(request):
if request.method == 'POST':
data = json.loads(request.body)
data['configurable'] = 1
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
cmd = 'cd ' + path + '&& scrapy startproject ' + data.get('name')
result = os.system(cmd)
if result == 0:
Project.objects.update_or_create(**data)
return HttpResponse(data.get('name'))
def index_status(request):
if request.method == 'GET':
clients = Client.objects.all()
data = {
'success': 0,
'error': 0,
'project': 0,
}
for client in clients:
try:
requests.get(scrapyd_url(client.ip, client.port), timeout=1)
data['success'] += 1
except ConnectionError:
data['error'] += 1
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
files = os.listdir(path)
for file in files:
if os.path.isdir(merge(path, file)) and not file in IGNORES:
data['project'] += 1
return HttpResponse(json.dumps(data))
def project_file_delete(request):
if request.method == 'POST':
data = json.loads(request.body)
path = merge(data['path'], data['label'])
result = os.remove(path)
return HttpResponse(json.dumps(result))
def project_file_create(request):
if request.method == 'POST':
data = json.loads(request.body)
path = merge(data['path'], data['name'])
open(path, 'w').close()
return HttpResponse('1')
def project_file(request):
if request.method == 'POST':
data = json.loads(request.body)
path = merge(data['path'], data['label'])
with open(path, 'r') as f:
return HttpResponse(f.read())
def project_tree(request, name):
if request.method == 'GET':
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
tree = get_tree(merge(path, name))
return HttpResponse(json.dumps(tree))
