def __make_json_file(self, file_path, version=None, message="no description..."):
# 创建json文件
json_file_path = self.__get_file_json_path(version)
# check file exists
if os.path.isfile(json_file_path):
logging.warning("{json_file_path} exists, do nothing...".format(
json_file_path=json_file_path))
return False
# get user
user = getpass.getuser()
# get current time
current_time = time.strftime("%Y-%m-%d %H:%M:%S")
# get file MD5
hash_code = get_md5(file_path)
# init info dict
info_dict = {}
info_dict["file_name"] = self.full_name
info_dict["message"] = message
info_dict["uploader"] = user
info_dict["upload_time"] = current_time
info_dict["file_MD5"] = hash_code
info_dict["version"] = version
# write info into json file
return self.__dict_to_json_file(info_dict, json_file_path)
def fetch_md5(self):
"""
:type self: object
"""
r = requests.get(self.http_md5,
auth=(self.user, self.password),
cert=(os.path.join(os.getcwd(), self.ssl_crt),
os.path.join(os.getcwd(), self.ssl_key)),
verify=False)
logger_settings.logger.info(u'Response code {0:d}'.format(
r.status_code))
r.raise_for_status()
logger_settings.logger.info('Response md5 from server {0}'.format(
r.content.rstrip('\n')))
if r.content.rstrip('\n') == getmd5.get_md5():
logger_settings.logger.info(
'Same config, doing nothing...sleeping')
else:
logger_settings.logger.info('New config detected, fun begins')
v = requests.get(self.openvpn_config_file,
auth=(self.user, self.password),
cert=(os.path.join(os.getcwd(), self.ssl_crt),
os.path.join(os.getcwd(), self.ssl_key)),
stream=True,
verify=False)
logger_settings.logger.info(u'Response code {0:d}'.format(
v.status_code))
v.raise_for_status()
with open(self.file_name_saved_local, 'wb') as handle:
for block in v.iter_content(1024):
handle.write(block)
logger_settings.logger.info(
'Moving new config to openvpn directory')
try:
shutil.copy(
self.file_name_saved_local,
os.path.join(os.getcwd(), self.openvpn_config_save_file))
logger_settings.logger.info('Restarting openvpn')
dist_name = platform.linux_distribution()[0]
mac_os = platform.system()
if dist_name.upper() in ['DEBIAN', 'UBUNTU']:
command = ['service', 'openvpn', 'restart']
else:
if mac_os == 'Darwin':
command = [
'/usr/local/opt/openvpn/sbin/openvpn',
'/etc/openvpn/client_danbordeanu'
]
else:
command = [
'systemctl', 'restart',
'openvpn@client_danbordeanu.service'
]
subprocess.call(command, shell=False)
except IOError, e:
logger_settings.logger.debug(
'Huston we have a big problem {0}'.format(e))
def verify_files(self, version=None):
# 检查文件MD5值是否有误
file_path = self.__get_file_json_path(version)
file_hash = get_md5(file_path)
version_dict = self.get_version_info(version)
json_hash = version_dict["file_MD5"]
if file_hash == json_hash:
logging.info("{file_path} checked. Result Match.")
return True
else:
logging.warning("{file_path} MD5 not Match.")
return False
def fetch_md5(self):
"""
:type self: object
"""
r = requests.get(self.http_md5, auth=(self.user, self.password), cert=(os.path.join(os.getcwd(), self.ssl_crt),
os.path.join(os.getcwd(),
self.ssl_key)),
verify=False)
logger_settings.logger.info(u'Response code {0:d}'.format(r.status_code))
r.raise_for_status()
logger_settings.logger.info('Response md5 from server {0}'.format(r.content.rstrip('\n')))
if r.content.rstrip('\n') == getmd5.get_md5():
logger_settings.logger.info('Same config, doing nothing...sleeping')
else:
logger_settings.logger.info('New config detected, fun begins')
v = requests.get(self.openvpn_config_file, auth=(self.user, self.password),
cert=(os.path.join(os.getcwd(), self.ssl_crt),
os.path.join(os.getcwd(), self.ssl_key)), stream=True,
verify=False)
logger_settings.logger.info(u'Response code {0:d}'.format(v.status_code))
v.raise_for_status()
with open(self.file_name_saved_local, 'wb') as handle:
for block in v.iter_content(1024):
handle.write(block)
logger_settings.logger.info('Moving new config to openvpn directory')
try:
shutil.copy(self.file_name_saved_local, os.path.join(os.getcwd(), self.openvpn_config_save_file))
logger_settings.logger.info('Restarting openvpn')
dist_name = platform.linux_distribution()[0]
mac_os = platform.system()
if dist_name.upper() in ['DEBIAN', 'UBUNTU']:
command = ['service', 'openvpn', 'restart']
else:
if mac_os == 'Darwin':
command = ['/usr/local/opt/openvpn/sbin/openvpn', '/etc/openvpn/client_danbordeanu']
else:
command = ['systemctl', 'restart', 'openvpn@client_danbordeanu.service']
subprocess.call(command, shell=False)
except IOError, e:
logger_settings.logger.debug('Huston we have a big problem {0}'.format(e))
def get_blog_info(num):
index_url = 'http://www.flightclub.cn/mapi/data/news_list_full/' + str(num * 30)
data = get_response(index_url)
info_json = json.loads(data)
redis_operate = RedisOperate()
temp = 0
if info_json['ret'] == 'ok':
print("爬取第" + str(num+1) + "页------------")
for blog in info_json['msg']['list']:
blog_url = urljoin(index_url, blog['url'])
url_md5 = get_md5(blog_url)
if not redis_operate.duplicate_checking(url_md5):
blog_id = blog['id']
blog_title = blog['title']
print('爬取' + blog_title)
blog_pic = urljoin(index_url, blog['pic'])
download_image(blog_id, blog_pic)
blog_image = 'blog/2020/02/' + blog_id + '.jpg'
blog_content = get_blog_content(blog_url)
blog_release_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
params = (blog_id, blog_title, blog_content, blog_image, blog_release_time, datetime.now())
try:
insert_sql = """
INSERT INTO blog_blog (`id`, `title`, `content`, `image`, `release_time`, `add_time`) VALUES (%s,%s,%s,%s,%s,%s)
"""
cursor.execute(insert_sql, params)
db.commit()
redis_operate.put(url_md5)
print('爬取' + blog_title + '成功')
except Exception as e:
print(e)
else:
temp += 1
if temp > 5:
quit()
print("重复url,跳过")
continue
def test_is_there_md5(self):
"""
Test if md5 working
:return:
"""
self.assertTrue(get_md5(), 'Uhh, md5 not working')
def compile_files(first_file, sec_file):
if os.path.isfile(first_file) and os.path.isfile(sec_file):
if get_md5.get_md5(first_file) == get_md5.get_md5(sec_file):
return True
return False
raise ValueError("not a exist file.")
def test_is_there_md5(self):
"""
Test if md5 working
:return:
"""
self.assertTrue(get_md5(), 'Uhh, md5 not working')