def mkdir_if_missing(path):
if not os.path.exists(path):
print('The path "' + path + '" does not exist. Try creating it.')
config.mkdir(path)
if not os.path.isdir(path):
print('ERROR: "' + path + '" is not a directory.')
return False
return True
def make_job(template_id, number, name):
code = True
message = u''
job_id = os.path.join(config.JOBS_ROOT, ("%04d-%s" % (number, name)))
try:
config.mkdir(job_id)
data = template.get_data(template_id)
data.update({'src': template_id})
job.set_data(job_id, data)
except TestsuiteError as e:
code = False
message = e.message
return code, message
def createstr(self, vhost):
schemes = ['http://']
if vhost.autossl:
schemes.append('https://')
# 生成配置文件
if vhost.domain.find(":") == -1:
domain = []
for scheme in schemes:
domain.append(scheme + vhost.domain)
else:
domain = [vhost.domain]
alias = vhost.alias.all()
for x in alias:
if x.domain.find(":") == -1:
for scheme in schemes:
domain.append(scheme + x.domain)
else:
domain.append(x.domain)
strlist = [' '.join(domain) + ' { ']
path = '%s/%s' % (WWW_DIR, vhost.root)
mkdir(path)
strlist.append('root /var/www/html/%s' % vhost.root)
if not vhost.autossl:
strlist.append('tls off')
if vhost.tls_crt != '' and vhost.tls_key != '':
strlist.append('tls /caddy/ssl/%s /caddy/ssl/%s' %
(vhost.tls_crt, vhost.tls_key))
if vhost.php:
strlist.append('fastcgi / %s:%s php' % (GATEWAY, vhost.php))
if vhost.gzip:
strlist.append('gzip')
if vhost.customize:
strlist.append(vhost.customize)
strlist.append('}')
string = ' \n'.join(strlist)
path = '%s/%s/vhosts/%s.conf' % (CONFIG_DIR, DRIVER_NAME, vhost.domain)
mkfile(path, string, True)
return string
def merge_dir(self, entry):
'''
A remote-first dir merge mechanism.
'''
local_path, remote_path = entry
# if the dir does not exist locally, try creating it
if not os.path.exists(local_path):
try: config.mkdir(local_path)
except OSError:
pass
config.log.info('syncing ' + local_path)
all_remote_items = self.api.list_entries(remote_path)
all_local_items = resolve_case_conflict(local_path)
for item in all_remote_items:
target_path = local_path + item['name']
item['parent_path'] = local_path
if is_ignorable(item['name']):
config.log.info('Skipped remote entry "' + item['name'] + '"')
pass
elif item['type'] in self.api.FOLDER_TYPES:
current_ent = self.find_entry(id = item['id'])
if current_ent != None and current_ent['status'] == 'moved_from':
self.sem.release()
continue
# if the item is a folder, resolve possible conflict
# and then add it to the bfs queue.
item['type'] = 'folder'
item['size'] = 0
item['status'] = 'synced'
if os.path.isfile(target_path):
confog.log.warning('"' + target_path + '" is a dir remotely, but file locally.')
try:
resolved_name = resolve_conflict(target_path)
all_local_items.append(resolved_name)
except OSError as e:
config.log.error('OSError {0}: {1}. Path: {2}.'.format(e.errno, e.strerr, e.filename))
continue
self.update_entry(item)
self.enqueue(target_path + '/', item['id'])
else:
current_ent = self.find_entry(id = item['id'])
if current_ent != None and current_ent['status'] == 'moved_from':
self.sem.release()
continue
# so the entry is a file
# first check if the local file exists
item['type'] = 'file'
if os.path.isfile(target_path):
# so both sides are files, compare their mtime
try:
local_mtime = config.timestamp_to_time(os.path.getmtime(target_path))
remote_mtime = config.str_to_time(item['client_updated_time'])
if local_mtime == remote_mtime:
# same files, same timestamp
# TODO: What if the files are different in content?
item['status'] = 'synced'
# config.log.debug(entry_local_path + ' was not changed. Skip.')
elif local_mtime > remote_mtime:
# local file is newer, upload it
row = self.find_entry(parent_path = local_path, name = item['name'])
if row != None and row['id'] == item['id'] and config.str_to_time(row['client_updated_time']) == remote_mtime:
# the file is changed offline, and the remote one wasn't changed before.
config.log.info('"' + target_path + ' is strictly newer. Upload it.')
item['status'] = 'put'
self.add_work('put', target_path, remote_parent_id = item['parent_id'], postwork='n')
else:
# the files are changed and no way to guarantee which one is abs newer.
config.log.info('"' + target_path + ' is newer but conflicts can exist. Rename the local file.')
item['status'] = 'get'
try:
new_name = resolve_conflict(target_path, config.OS_HOSTNAME + ', newer')
all_local_items.append(new_name)
self.add_work('get', target_path, remote_id = item['id'], postwork = 'n')
except OSError as e:
config.log.error('OSError {0}: {1}. Path: {2}.'.format(e.errno, e.strerr, e.filename))
else:
# in this branch the local file is older.
row = self.find_entry(parent_path = local_path, name = item['name'])
if row != None and row['id'] == item['id'] and config.str_to_time(row['client_updated_time']) == local_mtime:
# so the remote file is absolutely newer
# the local file wasn't changed when the program is off.
config.log.info('"' + target_path + ' is strictly older. Download it.')
item['status'] = 'get'
self.add_work('get', target_path, remote_id = item['id'], postwork = 'u')
else:
config.log.info('"' + target_path + ' is older but conflicts can exist. Rename the local file.')
item['status'] = 'get'
try:
new_name = resolve_conflict(target_path, config.OS_HOSTNAME + ', older')
all_local_items.append(new_name)
self.add_work('get', target_path, remote_id = item['id'], postwork = 'n')
except OSError as e:
config.log.error('OSError {0}: {1}. Path: {2}.'.format(e.errno, e.strerr, e.filename))
except OSError as e:
config.log.error('OSError {0}: {1}. Path: {2}.'.format(e.errno, e.strerr, e.filename))
elif os.path.isdir(target_path):
# the local file is a dir but the remote path is a file
resolved_name = resolve_conflict(target_path)
if resolved_name != None:
all_local_items.append(resolved_name)
itme['status'] = 'get'
self.add_work('get', target_path, remote_id = item['id'], postwork = 'n')
else:
config.log.critical('"' + target_path + '" cannot sync because of non-resolvable type conflict.')
continue
else:
# so the file does not exist locally.
item['status'] = 'get'
self.add_work('get', target_path, remote_id = item['id'], postwork = 'n')
self.update_entry(item)
if item['name'] in all_local_items:
all_local_items.remove(item['name'])
# process untouched local items
for name in all_local_items:
if is_ignorable(name): continue
entry_record = self.find_entry(parent_path = local_path, name = name)
if entry_record != None:
# the entry exists physically and in db, but not touched remotely
# then the file is probably deleted remotely
config.log.debug('The entry "' + local_path + name + '" may have been deleted remotely but not reflected locally.')
self.add_work('clean', local_path + name, postwork = 'd')
elif os.path.isdir(local_path + name):
config.log.debug('Local dir "' + local_path + name + '" does not exist remotely. Create it.')
self.remote_mkdir(local_path, name, remote_path)
else:
config.log.debug('Local file "' + local_path + name + '" does not exist remotely. Upload it.')
self.add_work('put', local_path + name, remote_parent_id = remote_path, postwork = 'n')
while self.pending_work_count > 0:
self.sem.release()
self.pending_work_count -= 1
def mkdir(self, path):
mkdir(path)
def createstr(self, vhost):
# 生成vhost字符串
strlist = []
# 主域名
strlist.append('ServerName %s' % vhost.domain)
# 别名
alias = vhost.alias.all()
if alias:
aliasdomain = []
for x in alias:
aliasdomain.append(x.domain)
strlist.append('ServerAlias %s' % ' '.join(aliasdomain))
if vhost.email != '':
strlist.append('ServerAdmin %s' % vhost.email)
# 根目录
path = '%s/%s' % (WWW_DIR, vhost.root)
mkdir(path)
strlist.append('DocumentRoot /var/www/html/%s' % vhost.root)
# php
if vhost.php:
strlist.append('<Proxy "fcgi://%s:%s/" enablereuse=on max=10>' %
(GATEWAY, vhost.php))
strlist.append('</Proxy>')
strlist.append('<FilesMatch "\\.php$">')
strlist.append('SetHandler "proxy:fcgi://%s:%s"' %
(GATEWAY, vhost.php))
strlist.append('</FilesMatch>')
strlist.append('AddType application/x-httpd-php .php')
strlist.append('DirectoryIndex index.html index.htm index.php')
# 反向代理
proxys = vhost.proxy.all()
for proxy in proxys:
strlist.append('<Location %s>' % proxy.from_path)
strlist.append('ProxyPass %s' % proxy.to_url)
strlist.append('ProxyPassReverse %s' % proxy.to_url)
strlist.append(
'RequestHeader append X-Forwarded-For %%{REMOTE_ADDR}s')
strlist.append('</Location>')
# 合并
base_string = '\n'.join(strlist)
strlist = [
'<VirtualHost *:80>',
base_string,
'</VirtualHost>',
]
# ssl
if vhost.tls_crt != '' and vhost.tls_key != '':
strlist.append('<VirtualHost *:443>')
strlist.append(base_string)
strlist.append('SSLEngine on')
strlist.append('SSLCertificateFile /usr/local/apache2/ssl/%s' %
vhost.tls_crt)
strlist.append('SSLCertificateKeyFile /usr/local/apache2/ssl/%s' %
vhost.tls_key)
if vhost.tls_chain != '':
strlist.append(
'SSLCertificateChainFile /usr/local/apache2/ssl/%s' %
vhost.tls_chain)
strlist.append('</VirtualHost>')
# 输出
string = ' \n'.join(strlist)
path = '%s/%s/vhosts/%s.conf' % (CONFIG_DIR, DRIVER_NAME, vhost.domain)
mkfile(path, string, True)
return string
import sys
sys.path.append("conf/")
import config as myconf
import threading
import time
import urllib
from urllib import request
from bs4 import BeautifulSoup
url = 'https://www.192tt.com/gc/vgirl/vgirl011.html'
# print(fixedurl[:-5])
fixedurl = url[:-5]
html = myconf.getHtml(url)
MainMessage = myconf.getHtmlMessage(html)
# print('Media/' + MainMessage['imgname'])
myconf.mkdir('Media/' + MainMessage['imgname'])
# print(message['imgurl'])
# soup = BeautifulSoup(html,"html5lib")
# soup = BeautifulSoup(html, "html.parser")
# fwrite(soup.prettify())
# print(soup.img['lazysrc'])
# print(soup.prettify())
# altstr = 'Rosimm.com - NO.2421[ROSI.CC - NO.2421]'
# imgurl = soup.img['lazysrc']
# imgname = soup.img['alt']
# imgurl = soup.id['allnum']
# imgnum = soup.find('span', id='allnum').get_text()
# print(imgnum,imgname,imgurl)
# 主程序
# fixedurl = 'https://www.192tt.com/gc/rosimm/rosi2421'
myconf.trunc_table()