def response_load():
# Verify files exist
for i in respfile:
if not (os.path.isfile(path + i)):
try:
logger.warning("Missing file " + i +
" , Downloading from repo.")
urllib.request.urlretrieve(baseurl + i, path + i)
except urllib.error.HTTPError as e:
logger.critical("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
logger.critical(e)
logger.critical(baseurl + i)
logger.critical("Resolve networking or create file. ")
logger.critical("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
quit(1)
# Load files to memory
index = 0
for i in respfile:
resparray = []
for line in open(path + respfile[index]):
# Quick and dirty fix for having newlines in quote files
line = line.strip('\n')
line = line.replace('\\n', '\n')
resparray.append(line)
respdata.append(resparray)
index += 1
def load_conf(self, section, mandatories=['host', 'port'], options=None):
"""Load the export <section> configuration in the Glances configuration file.
:param section: name of the export section to load
:param mandatories: a list of mandatories parameters to load
:param options: a list of optionnals parameters to load
:returns: Boolean -- True if section is found
"""
options = options or []
if self.config is None:
return False
# By default read the mandatory host:port items
try:
for opt in mandatories:
setattr(self, opt, self.config.get_value(section, opt))
except NoSectionError:
logger.critical("No {} configuration found".format(section))
return False
except NoOptionError as e:
logger.critical("Error in the {} configuration ({})".format(section, e))
return False
# Load options
for opt in options:
try:
setattr(self, opt, self.config.get_value(section, opt))
except NoOptionError:
pass
logger.debug("Load {} from the Glances configuration file".format(section))
logger.debug("{} parameters: {}".format(section, {opt: getattr(self, opt) for opt in mandatories + options}))
return True
def create_or_update_workspace(*,
files: List[FilePath],
workspace_id: UUID4 = None) -> UUID4:
url = f'{get_gateway_url()}/workspaces'
with ExitStack() as file_stack:
files = [
('files',
file_stack.enter_context(open(f'{RENDER_DIR}/{file.name}', 'rb')))
for file in files
]
logger.debug(f'Creating a workspace to upload files: {files}')
if workspace_id:
r = requests.post(url=url,
data={'workspace_id': workspace_id},
files=files)
else:
r = requests.post(url=url, files=files)
if r.status_code != requests.codes.created:
logger.critical(
f'❌ Workspace creation failed with status {r.status_code}. Response: {r.json()}'
)
r.raise_for_status()
workspace_id = r.json()
logger.info(
f'💪 Successfully uploaded files to workspace: {workspace_id}')
return uuid.UUID(workspace_id)
async def reset_guild(self, message):
logger.critical("Resetting game guild !")
await self._guild_manager.reset_all_channels(message)
channel = await get_safe_text_channel(guild=message.guild,
name_key="BOARD")
await self.show_control_panel(message.guild, channel)
logger.info("Game guild reset !")
def setup_down_path():
'''设置图片下载后的保存位置，所有图片放在同一个目录下'''
down_path = os.path.join(basepath, 'downloads')
if not os.path.isdir(down_path):
os.mkdir(down_path)
logger.critical('Create download path {}'.format(down_path))
return down_path
def read_conf(self):
'''
Reads in the given configuration file.
Return True if successful, False otherwise.
'''
self._response_from_addr = settings.RESPONSE_FROM_ADDR
try:
with open(
aws_helpers.get_s3_cached_filepath(
settings.ATTACHMENT_CACHE_DIR,
settings.CONFIG_S3_BUCKET,
settings.CONFIG_S3_KEY)) as conffile:
# Note that json.load reads in unicode strings.
self._conf = json.load(conffile)
all_email_addrs = set()
# Do some validation
for item in self._conf:
if 'email_addr' not in item \
or 'body' not in item \
or 'attachments' not in item:
raise Exception('invalid config item: %s' % repr(item))
all_email_addrs.add(item['email_addr'])
except Exception as ex:
logger.critical('error: config file read failed: %s; file: %s:%s',
ex, settings.CONFIG_S3_BUCKET,
settings.CONFIG_S3_KEY)
return False
return True
async def step03():
# 用于限制并发请求数量，该网站并发能力太差，尽量不要设置得过大
sem3 = asyncio.Semaphore(64)
# 不能为每个请求创建一个seesion，减少开销
async with aiohttp.ClientSession() as session:
t3 = time.time()
ignored_image_pages = collection_image_pages.count_documents(
{'visited': 1}) # 被忽略的图片页面数
visited_image_pages = 0 # 请求成功的图片页面数
failed_image_pages = 0 # 请求失败的图片页面数
image_pages = collection_image_pages.find({'visited':
0}) # 所有未访问过的图片页面
to_do = [
get_image_url(sem3, session, image_page)
for image_page in image_pages
]
to_do_iter = asyncio.as_completed(to_do)
with progressbar.ProgressBar(max_value=len(to_do)) as bar:
for i, future in enumerate(to_do_iter):
result = await future
if result.get('failed'):
failed_image_pages += 1
else:
visited_image_pages += 1
bar.update(i)
logger.critical(
'Ignored [{}] image pages, visited [{}] image pages, failed [{}] image pages'
.format(ignored_image_pages, visited_image_pages,
failed_image_pages))
logger.critical('Step 03 cost {:.2f} seconds'.format(time.time() - t3))
await session.close()
def search_parser(keyword, page):
logger.debug('Searching doujinshis of keyword {}'.format(keyword))
result = []
try:
response = request('get',
url=constant.SEARCH_URL,
params={
'q': keyword,
'page': page
}).content
except requests.ConnectionError as e:
logger.critical(e)
logger.warn(
'If you are in China, please configure the proxy to fu*k GFW.')
raise SystemExit
html = BeautifulSoup(response)
doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
for doujinshi in doujinshi_search_result:
doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'})
title = doujinshi_container.text.strip()
title = (title[:85] + '..') if len(title) > 85 else title
id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1)
result.append({'id': id_, 'title': title})
return result
def read_conf(self):
'''
Reads in the given configuration file.
Return True if successful, False otherwise.
'''
self._response_from_addr = settings.RESPONSE_FROM_ADDR
try:
with open(aws_helpers.get_s3_cached_filepath(
settings.ATTACHMENT_CACHE_DIR,
settings.CONFIG_S3_BUCKET,
settings.CONFIG_S3_KEY)) as conffile:
# Note that json.load reads in unicode strings.
self._conf = json.load(conffile)
all_email_addrs = set()
# Do some validation
for item in self._conf:
if 'email_addr' not in item \
or 'body' not in item \
or 'attachments' not in item:
raise Exception('invalid config item: %s' % repr(item))
all_email_addrs.add(item['email_addr'])
except Exception as ex:
logger.critical('error: config file read failed: %s; file: %s:%s', ex, settings.CONFIG_S3_BUCKET, settings.CONFIG_S3_KEY)
return False
return True
def db_get_channel_route(s_plz, try_cache=True):
if try_cache:
try:
ch_id = PLZ_CH_ID_DICT[s_plz]
logger.info("Channel-ID %s for PLZ %s loaded get cache", ch_id,
s_plz)
return ch_id
except KeyError:
logger.warning(
"Couldn't find PLZ %s in cache, try to fetch from DB", s_plz)
pass
sql_str = "SELECT plz_kv.plz, kv_ch_id_hessen.ch_id from plz_kv " \
"inner join kv_ch_id_hessen on plz_kv.kreis = kv_ch_id_hessen.kv " \
"where plz = %s"
try:
with PSCON as con:
with con.cursor() as c:
c.execute(sql_str, (s_plz, ))
res = c.fetchone()
logger.info("Get %s from DB", res)
except Exception as e:
logger.critical("DB-Error %s", e)
res = None
return res
if res is None:
print("here")
return
else:
PLZ_CH_ID_DICT[res[0]] = res[1] # Update Cache
logger.info("Updated cached channel-id of plz %s to %s", res[0],
res[1])
return res[1]
def download(self, queue, folder=''):
if not isinstance(folder, (str, unicode)):
folder = str(folder)
if self.path:
folder = os.path.join(self.path, folder)
if not os.path.exists(folder):
logger.warn('Path \'%s\' not exist.' % folder)
try:
os.mkdir(folder)
except EnvironmentError as e:
logger.critical('Error: %s' % str(e))
raise SystemExit
else:
logger.warn('Path \'%s\' already exist.' % folder)
queue = [([url], {'folder': folder}) for url in queue]
self.thread_pool = threadpool.ThreadPool(self.thread_count)
requests_ = threadpool.makeRequests(self._download, queue,
self._download_callback)
[self.thread_pool.putRequest(req) for req in requests_]
self.thread_pool.wait()
def create_client(self, type_client='app'):
"""
Create the client http://arxiv.org/pdf/1304.6257.pdfwith the auth keys. Works differently
if it is app_only or user_auth
Params:
*type_client: app, user
"""
client_key = self.twitter_keys['CONSUMER_KEY']
client_secret = self.twitter_keys['CONSUMER_SECRET']
try:
if type_client == 'user':
token_access = self.twitter_keys['ACCESS_TOKEN']
token_secret = self.twitter_keys['ACCESS_TOKEN_SECRET']
client = OAuth1Session(client_key,
client_secret=client_secret,
resource_owner_key=token_access,
resource_owner_secret=token_secret)
return client
elif type_client == 'app':
TOKEN_URL = 'https://api.twitter.com/oauth2/token'
client = BackendApplicationClient(client_id=client_key)
oauth = OAuth2Session(client=client)
token = oauth.fetch_token(token_url=TOKEN_URL,
client_id=client_key,
client_secret=client_secret)
return oauth
except AttributeError:
logger.critical('No Keys to connect, check the file')
raise "No Keys to connect, check the file"
def step04():
t4 = time.time()
visited_images = 0 # 请求成功的图片数
failed_images = 0 # 请求失败的图片数
if RE_DOWN:
ignored_images = 0 # 被忽略的图片数
images = collection_images.find()
max_value = collection_images.count_documents({})
else:
ignored_images = collection_images.count_documents({'visited':
1}) # 被忽略的图片数
images = collection_images.find({'visited': 0}) # 所有未访问过的图片URL
max_value = collection_images.count_documents({'visited': 0})
with progressbar.ProgressBar(max_value=max_value) as bar:
for i, image in enumerate(images):
result = download_image(image)
if result.get('ignored'):
ignored_images += 1
else:
if result.get('failed'):
failed_images += 1
else:
visited_images += 1
bar.update(i)
logger.critical(
'Ignored [{}] images, visited [{}] images, failed [{}] images'.format(
ignored_images, visited_images, failed_images))
logger.critical('Step 04 cost {:.2f} seconds'.format(time.time() - t4))
def _download(self, url, folder='', filename='', retried=False):
logger.info('Start downloading: %s ...' % url)
filename = filename if filename else os.path.basename(
urlparse(url).path)
try:
with open(os.path.join(folder, filename), "wb") as f:
response = requests.get(url, stream=True, timeout=self.timeout)
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(2048):
f.write(chunk)
except requests.HTTPError as e:
if not retried:
logger.error('Error: %s, retrying' % str(e))
return self._download(url=url,
folder=folder,
filename=filename,
retried=True)
else:
return None
except Exception as e:
logger.critical('CRITICAL: %s' % str(e))
return None
return url
def step03():
t3 = time.time()
ignored_image_pages = collection_image_pages.count_documents(
{'visited': 1}) # 被忽略的图片页面数
visited_image_pages = 0 # 请求成功的图片页面数
failed_image_pages = 0 # 请求失败的图片页面数
image_pages = collection_image_pages.find({'visited': 0}) # 所有未访问过的图片页面
workers = min(64, collection_image_pages.count_documents({'visited': 0}))
workers = workers if workers > 0 else 1
with futures.ThreadPoolExecutor(workers) as executor:
to_do = []
# 创建并排定Future
for image_page in image_pages:
future = executor.submit(get_image, image_page)
to_do.append(future)
# 获取Future的结果，futures.as_completed(to_do)的参数是Future列表，返回迭代器，
# 只有当有Future运行结束后，才产出future
done_iter = futures.as_completed(to_do)
with progressbar.ProgressBar(max_value=len(to_do)) as bar:
for i, future in enumerate(
done_iter
): # future变量表示已完成的Future对象，所以后续future.result()绝不会阻塞
result = future.result()
if result.get('failed'):
failed_image_pages += 1
else:
visited_image_pages += 1
bar.update(i)
logger.critical(
'Ignored [{}] image pages, visited [{}] image pages, failed [{}] image pages'
.format(ignored_image_pages, visited_image_pages, failed_image_pages))
logger.critical('Step 03 cost {:.2f} seconds'.format(time.time() - t3))
def new(cls, user_id, article_id, timestamp=None):
"""
Create new saves object and commit to db.
Return saves object if successful, otherwise return None.
Note: saves object can only be added internally, no UI.
"""
new_saves = cls(user_id=user_id,
article_id=article_id,
timestamp=timestamp)
try:
db.session.add(new_saves)
db.session.commit()
except IntegrityError as e:
logger.error(
f"Cannot add {new_saves} to database. {e.orig.pgerror}")
db.session.rollback()
return None
except SQLAlchemyError:
logger.critical(f'Failed to create {new_saves} on database.')
db.session.rollback()
return None
return new_saves
def register(cls, username, password, email, first_name, last_name):
"""
Register user w/hashed password & commit to db.
Return user object if successful, otherwise return None.
"""
hashed = cls.bcrypt.generate_password_hash(password)
# turn bytestring into normal (unicode utf8) string
hashed_utf8 = hashed.decode("utf8")
new_user = cls(username=username,
password=hashed_utf8,
email=email,
first_name=first_name,
last_name=last_name)
try:
db.session.add(new_user)
db.session.commit()
except IntegrityError:
flash("Username/email already exist.", "danger")
db.session.rollback()
return None
except SQLAlchemyError:
logger.critical(f'Failed to create {new_user} on database.')
flash(f"Failed to create '{username}'", "danger")
db.session.rollback()
return None
return new_user
def update(cls, old_username, new_username, password):
"""
Update user with specified id & commit to db.
Return user object if successful, otherwise return None.
"""
user = User.authenticate(old_username, password)
if not user:
flash("Username and password do not match!", 'danger')
return None
user.username = new_username
try:
db.session.add(user)
db.session.commit()
except IntegrityError:
flash("Username already exists.", "danger")
db.session.rollback()
return None
except SQLAlchemyError:
logger.critical(f'Failed to update {user} on database.')
flash(f"Failed to update '{username}'", "danger")
db.session.rollback()
return None
return user
def get_albums(url):
'''请求入口页面，BeautifulSoup解析返回的HTTP响应，从中获取所有图集的URL
:return:
find_albums: 本次请求入口页面后，一共找到多个图集
new_albums: 有多少个是新增的（即数据库之前没有记录的）
'''
find_albums = 0 # 一共找到多个图集
new_albums = 0 # 有多少个是新增的（即数据库之前没有记录的）
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
}
# 返回入口页面的HTTP响应
# TODO: 捕获request.get方法的异常，比如连接超时、被拒绝等
resp = get_response(url, info='start url', headers=headers)
if not resp: # 请求失败时，resp为None，不能往下执行
logger.error('Exception {} on URL [{}]'.format(e.__class__, url))
return
# 使用lxml解析器，解析返回的响应（HTML文档）
soup = BeautifulSoup(resp.text, 'lxml')
# 每个图集按年份/月份被放在 <div class='all'></div> 下面的每个<a href="图集URL">图集标题<a> 中
try:
a_tags = soup.find('div', {'class': 'all'}).find_all('a') # <class 'bs4.element.ResultSet'>
except Exception as e:
logger.error('Exception {} on URL [{}]'.format(e.__class__, url))
return
logger.critical('URL [{}] has [{}] <a></a> tags'.format(url, len(a_tags)))
for a in a_tags:
# 判断每个<a></a>标签中的URL是不是符合图集URL格式，如果不是，则递归调用它看看它下面有没有相同URL
# 因为有一个 https://www.mzitu.com/old/
# 在 https://www.mzitu.com/old/ 页面中第一个URL 又是 https://www.mzitu.com/all/ 需要排除它，不然就无限死循环了
if a['href'] == 'https://www.mzitu.com/all/':
continue
if re.match(r'https://www.mzitu.com/\d+', a['href']):
data = {
'album_title': a.get_text(), # 每个图集的标题
'album_url': a['href'], # 每个图集的URL
'created_at': datetime.utcnow(), # 何时添加到MongoDB的
'visited': 0 # 表明此图集URL没被访问过
}
if TEST_NUM and TEST_NUM == collection_albums.count_documents({}): # 如果是测试，MongoDB中永远只能存在 TEST_NUM 个集合
find_albums += 1 # 增加找到的图集计数
continue
else:
if not collection_albums.find_one({'album_url': data['album_url']}): # 如果是新图集，保存到MongoDB
new_albums += 1 # 增加新图集计数
collection_albums.insert_one(data)
logger.debug('Successfully saved album {} [{}] to MongoDB'.format(data['album_title'], data['album_url']))
find_albums += 1 # 增加找到的图集计数
else: # 此<a></a>标签包含的URL不是一个合法的图集地址，即将递归获取它下面的图集
logger.critical('Tag [<a href="{}">{}</a>] is invalid, recursive call this function'.format(a['href'], a.get_text()))
recursive_find_albums, recursive_new_albums = get_albums(a['href'])
find_albums += recursive_find_albums
new_albums += recursive_new_albums
return find_albums, new_albums
def get_data_or_die(filename, mds_header, hmap, all_eps=None):
data = read_data(filename, mds_header, hmap, all_eps=all_eps)
if not data or not data['episodes'] or len(data['episodes']) < 1:
logger.critical("No data. Quitting...")
sys.exit(0)
return data
def __post_init__(self) -> NoReturn:
if not self.is_online():
logger.critical(f'client: {self.ip} is down!')
# sys.exit(1)
if self.debug:
paramiko.util.log_to_file('paramiko.log')
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
def doujinshi_parser(id_):
if not isinstance(id_,
(int, )) and (isinstance(id_,
(str, )) and not id_.isdigit()):
raise Exception('Doujinshi id({}) is not valid'.format(id_))
id_ = int(id_)
logger.log(15, 'Fetching doujinshi information of id {}'.format(id_))
doujinshi = dict()
doujinshi['id'] = id_
url = '{}/{}/'.format(constant.DETAIL_URL, id_)
try:
response = request('get', url).content
except Exception as e:
logger.critical(str(e))
sys.exit()
html = BeautifulSoup(response)
doujinshi_info = html.find('div', attrs={'id': 'info'})
title = doujinshi_info.find('h1').text
subtitle = doujinshi_info.find('h2')
doujinshi['name'] = title
doujinshi['subtitle'] = subtitle.text if subtitle else ''
doujinshi_cover = html.find('div', attrs={'id': 'cover'})
img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$',
doujinshi_cover.a.img['src'])
if not img_id:
logger.critical('Tried yo get image id failed')
sys.exit()
doujinshi['img_id'] = img_id.group(1)
doujinshi['ext'] = img_id.group(2)
pages = 0
for _ in doujinshi_info.find_all('div', class_=''):
pages = re.search('([\d]+) pages', _.text)
if pages:
pages = pages.group(1)
break
doujinshi['pages'] = int(pages)
# gain information of the doujinshi
information_fields = doujinshi_info.find_all('div',
attrs={'class': 'field-name'})
needed_fields = ['Characters', 'Artists', 'Language', 'Tags']
for field in information_fields:
field_name = field.contents[0].strip().strip(':')
if field_name in needed_fields:
data = [
sub_field.contents[0].strip()
for sub_field in field.find_all('a', attrs={'class': 'tag'})
]
doujinshi[field_name.lower()] = ', '.join(data)
return doujinshi
def record_tweet(self):
""" Record the tweet in MongoDB
"""
try:
if self.tweetdict:
self.collection.insert(self.tweetdict)
# FIXME Need to check the right error here - too vague, bug in writing body
except Exception as e:
logger.critical('{}: Error in inserting tweet in db: {}'.format(datetime.datetime.now(), e))
def download(self):
logger.info('Start download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
for i in xrange(1, self.pages + 1):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
self.downloader.download(download_queue, self.id)
else:
logger.critical('Downloader has not be loaded')
def delete_workspace(*, workspace_id: UUID4):
url = f'{get_gateway_url()}/workspaces'
r = requests.delete(url=f'{url}/{workspace_id}')
if r.status_code != requests.codes.ok:
logger.critical(
f'❌ Workspace deletion failed with status {r.status_code}. Response: {r.json()}'
)
r.raise_for_status()
logger.info(f'💪 Successfully deleted workspace {workspace_id}')
def create_cert(self, body):
"""Create a certificate."""
lifetime = int(body["lifetime"])
min_lifetime = int(self.config.get("mtls", "min_lifetime",
fallback=60))
max_lifetime = int(self.config.get("mtls", "max_lifetime", fallback=0))
if lifetime < min_lifetime:
logger.info(
"User requested lifetime less than minimum. {} < {}".format(
lifetime, min_lifetime))
error_response("lifetime must be greater than {} seconds".format(
min_lifetime))
if max_lifetime != 0:
if lifetime > max_lifetime:
logger.info(
"User requested lifetime greater than maximum. {} < {}".
format(lifetime, max_lifetime))
error_response("lifetime must be less than {} seconds".format(
max_lifetime))
csr_str = body["csr"]
csr = self.cert_processor.get_csr(csr_str)
if csr is None:
return error_response("Could not load CSR")
try:
csr_public_bytes = csr.public_bytes(serialization.Encoding.PEM)
sig_path = write_sig_to_file(body["signature"])
fingerprint = self.cert_processor.verify(csr_public_bytes,
sig_path)
os.remove(sig_path)
except CertProcessorUntrustedSignatureError as e:
logger.info("Unauthorized: {}".format(e))
return error_response("Unauthorized", 403)
except CertProcessorInvalidSignatureError:
logger.info("Invalid signature in CSR.")
return error_response("Invalid signature", 401)
if csr is None:
logger.info("Invalid CSR.")
return error_response("Invalid CSR")
cert = None
try:
cert = self.cert_processor.generate_cert(csr, lifetime,
fingerprint)
return json.dumps({"cert": cert.decode("UTF-8")}), 200
except CertProcessorKeyNotFoundError:
logger.critical("Key missing. Service not properly initialized")
return error_response("Internal Error")
except CertProcessorMismatchedPublicKeyError:
logger.error("CSR Public Key does not match found certificate.")
return error_response("Internal Error")
except CertProcessorNotAdminUserError:
logger.error(
"User {} is not an admin and attempted ".format(fingerprint) +
"to generate a certificate they are not allowed to generate.")
return error_response("Invalid Request", 403)
except CertProcessorNoPGPKeyFoundError:
logger.info("PGP Key not found.")
return error_response("Unauthorized", 401)
def db_get_all_channel_id():
sql_str = "Select ch_id from kv_ch_id_hessen"
try:
with PSCON as con:
with con.cursor() as c:
c.execute(sql_str)
res = c.fetchall()
except Exception as e:
logger.critical("DB-Error: %s", e)
return res
def download(self):
logger.info('Start download dojinshi: %s' % self.name)
if self.downloader:
download_queue = []
for i in xrange(1, self.pages + 1):
download_queue.append(
'%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
self.downloader.download(download_queue, self.id)
else:
logger.critical('Downloader has not be loaded')
def go():
'''
Reads in the given configuration file.
Return True if successful, False otherwise.
'''
try:
# Note that json.load reads in unicode strings.
conf_data, new_conf = aws_helpers.get_s3_cached_file(
settings.ATTACHMENT_CACHE_DIR, settings.CONFIG_S3_BUCKET,
settings.CONFIG_S3_KEY)
conf_data = json.loads(conf_data.read())
all_email_addrs = set()
# Do some validation
for item in conf_data:
if 'email_addr' not in item \
or 'body' not in item \
or 'attachments' not in item:
raise Exception('invalid config item: %s' % repr(item))
all_email_addrs.add(item['email_addr'])
if new_conf:
# Write the supported domains to files that will be used by
# Postfix in its config.
email_domains = set(
[addr[addr.find('@') + 1:] for addr in all_email_addrs])
with open(RESPONDER_DOMAINS_LIST_FILE,
'w') as responder_domains_file:
responder_domains_file.write(' '.join(email_domains))
address_maps_lines = [
'%s\t\t%s@localhost' % (addr, settings.MAIL_RESPONDER_USERNAME)
for addr in all_email_addrs
]
catchall_lines = [
'@%s\t\t%s@localhost' % (domain, settings.SYSTEM_DEVNULL_USER)
for domain in email_domains
]
with open(ADDRESS_MAPS_LIST_FILE, 'w') as address_maps_file:
address_maps_file.write('\n'.join(address_maps_lines))
address_maps_file.write('\n')
address_maps_file.write('\n'.join(catchall_lines))
address_maps_file.write('\n')
except Exception as ex:
print('error: config file pull failed: %s; file: %s:%s' %
(ex, settings.CONFIG_S3_BUCKET, settings.CONFIG_S3_KEY))
logger.critical('error: config file pull failed: %s; file: %s:%s', ex,
settings.CONFIG_S3_BUCKET, settings.CONFIG_S3_KEY)
return False
return True
def step01():
t1 = time.time()
# 入口页面
start_url = 'http://www.mzitu.com/all/'
# 访问入口页面，将所有图集信息保存到MongoDB
result = get_albums(start_url)
if not result: # 如果访问入口页面失败，则结束整个应用
return
else:
logger.critical('Find [{}] albums, insert [{}] new albums into MongoDB'.format(result[0], result[1]))
logger.critical('Step 01 cost {:.2f} seconds'.format(time.time() - t1))
def run():
"Startet den Server."
if not Lang.query.first():
logger.critical("Missing Lang(s)")
exit()
app_run_params = dict(host="::", port=env.web_port, threaded=True)
if env.DEBUG and not env.USE_RELOADER:
app_run_params["use_reloader"] = False
app.run(**app_run_params)
async def init_bot():
"""Coroutine to initialize the bot once it is ready for the first time."""
logger.info("Bot initialization")
# Add the enum of all available listeners
MinigameCollection.set_listener_enum(ListenersEnum)
# Initialize all available guilds
is_ok = await GuildManager().init_guilds(GAME_LANGUAGE)
if is_ok:
logger.info("Bot initialized correctly!")
else:
logger.critical("Bot initialization failure!")
def terminate_flow(*, flow_id: UUID4, delete_workspace: bool = False):
url = f'{get_gateway_url()}/flows'
r = requests.delete(url=f'{url}/{flow_id}',
json={'workspace': delete_workspace})
if r.status_code != requests.codes.ok:
logger.critical(
f'❌ Flow termination failed with status {r.status_code}. Response: {r.json()}'
)
r.raise_for_status()
logger.info(f'💪 Successfully terminated flow {flow_id}')
def doujinshi_parser(id_):
if not isinstance(id_, (int,)) and (isinstance(id_, (str,)) and not id_.isdigit()):
raise Exception('Doujinshi id({}) is not valid'.format(id_))
id_ = int(id_)
logger.log(15, 'Fetching doujinshi information of id {}'.format(id_))
doujinshi = dict()
doujinshi['id'] = id_
url = '{}/{}/'.format(constant.DETAIL_URL, id_)
try:
response = request('get', url).content
except Exception as e:
logger.critical(str(e))
sys.exit()
html = BeautifulSoup(response)
doujinshi_info = html.find('div', attrs={'id': 'info'})
title = doujinshi_info.find('h1').text
subtitle = doujinshi_info.find('h2')
doujinshi['name'] = title
doujinshi['subtitle'] = subtitle.text if subtitle else ''
doujinshi_cover = html.find('div', attrs={'id': 'cover'})
img_id = re.search('/galleries/([\d]+)/cover\.(jpg|png)$', doujinshi_cover.a.img['src'])
if not img_id:
logger.critical('Tried yo get image id failed')
sys.exit()
doujinshi['img_id'] = img_id.group(1)
doujinshi['ext'] = img_id.group(2)
pages = 0
for _ in doujinshi_info.find_all('div', class_=''):
pages = re.search('([\d]+) pages', _.text)
if pages:
pages = pages.group(1)
break
doujinshi['pages'] = int(pages)
# gain information of the doujinshi
information_fields = doujinshi_info.find_all('div', attrs={'class': 'field-name'})
needed_fields = ['Characters', 'Artists', 'Language', 'Tags']
for field in information_fields:
field_name = field.contents[0].strip().strip(':')
if field_name in needed_fields:
data = [sub_field.contents[0].strip() for sub_field in
field.find_all('a', attrs={'class': 'tag'})]
doujinshi[field_name.lower()] = ', '.join(data)
return doujinshi
def get_new_values(self):
try:
new_info = self.stream_new_info.get()
self.loop_number = new_info['loop_number']
self.keyterm = new_info['lvl1'] + new_info['lvl2']
if len(self.keyterm) > 5000:
self.keyterm = random.sample(self.keyterm, 5000)
logger.info('Get new information - nbr of id {} - loop nbr {}'.format(len(self.keyterm),
self.loop_number))
except Exception as e:
logger.critical('error in getting new values - {} - {}'.format(new_info, e))
raise Exception(e)
def run():
"Startet den Server."
if not (Lang.query.first() and GameType.query.first()):
logger.critical("Missing Lang(s) / GameType(s)")
exit()
app_run_params = dict(host="::", port=env.web_port, threaded=True)
if env.DEBUG and not env.USE_RELOADER:
app_run_params["use_reloader"] = False
if env.ssl:
import ssl
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
context.load_cert_chain('server.crt', 'server.key')
app_run_params["ssl_context"] = context
app.run(**app_run_params)
def go():
'''
Reads in the given configuration file.
Return True if successful, False otherwise.
'''
try:
# Note that json.load reads in unicode strings.
conf_data, new_conf = aws_helpers.get_s3_cached_file(
settings.ATTACHMENT_CACHE_DIR,
settings.CONFIG_S3_BUCKET,
settings.CONFIG_S3_KEY)
conf_data = json.loads(conf_data.read())
all_email_addrs = set()
# Do some validation
for item in conf_data:
if 'email_addr' not in item \
or 'body' not in item \
or 'attachments' not in item:
raise Exception('invalid config item: %s' % repr(item))
all_email_addrs.add(item['email_addr'])
if new_conf:
# Write the supported domains to files that will be used by
# Postfix in its config.
email_domains = set([addr[addr.find('@')+1:] for addr in all_email_addrs])
with open(RESPONDER_DOMAINS_LIST_FILE, 'w') as responder_domains_file:
responder_domains_file.write(' '.join(email_domains))
address_maps_lines = ['%s\t\t%s@localhost' % (addr, settings.MAIL_RESPONDER_USERNAME) for addr in all_email_addrs]
catchall_lines = ['@%s\t\t%s@localhost' % (domain, settings.SYSTEM_DEVNULL_USER) for domain in email_domains]
with open(ADDRESS_MAPS_LIST_FILE, 'w') as address_maps_file:
address_maps_file.write('\n'.join(address_maps_lines))
address_maps_file.write('\n')
address_maps_file.write('\n'.join(catchall_lines))
address_maps_file.write('\n')
except Exception as ex:
print('error: config file pull failed: %s; file: %s:%s' % (ex, settings.CONFIG_S3_BUCKET, settings.CONFIG_S3_KEY))
logger.critical('error: config file pull failed: %s; file: %s:%s', ex, settings.CONFIG_S3_BUCKET, settings.CONFIG_S3_KEY)
return False
return True
def search_parser(keyword, page):
logger.debug('Searching doujinshis of keyword {}'.format(keyword))
result = []
try:
response = request('get', url=constant.SEARCH_URL, params={'q': keyword, 'page': page}).content
except requests.ConnectionError as e:
logger.critical(e)
logger.warn('If you are in China, please configure the proxy to fu*k GFW.')
raise SystemExit
html = BeautifulSoup(response)
doujinshi_search_result = html.find_all('div', attrs={'class': 'gallery'})
for doujinshi in doujinshi_search_result:
doujinshi_container = doujinshi.find('div', attrs={'class': 'caption'})
title = doujinshi_container.text.strip()
title = (title[:85] + '..') if len(title) > 85 else title
id_ = re.search('/g/(\d+)/', doujinshi.a['href']).group(1)
result.append({'id': id_, 'title': title})
return result
def dojinshi_parser(id):
if not isinstance(id, (int,)) and (isinstance(id, (str,)) and not id.isdigit()):
raise Exception("Dojinshi id(%s) is not valid" % str(id))
id = int(id)
logger.debug("Fetching dojinshi information of id %d" % id)
dojinshi = dict()
dojinshi["id"] = id
url = "%s/%d/" % (DETAIL_URL, id)
try:
response = requests.get(url).content
except Exception as e:
logger.critical("%s%s" % tuple(e.message))
sys.exit()
html = BeautifulSoup(response)
dojinshi_info = html.find("div", attrs={"id": "info"})
title = dojinshi_info.find("h1").text
subtitle = dojinshi_info.find("h2")
dojinshi["name"] = title
dojinshi["subtitle"] = subtitle.text if subtitle else ""
dojinshi_cover = html.find("div", attrs={"id": "cover"})
img_id = re.search("/galleries/([\d]+)/cover\.(jpg|png)$", dojinshi_cover.a.img["src"])
if not img_id:
logger.critical("Tried yo get image id failed")
sys.exit()
dojinshi["img_id"] = img_id.group(1)
dojinshi["ext"] = img_id.group(2)
pages = 0
for _ in dojinshi_info.find_all("div", class_=""):
pages = re.search("([\d]+) pages", _.text)
if pages:
pages = pages.group(1)
break
dojinshi["pages"] = int(pages)
return dojinshi
def cmd_parser():
parser = OptionParser()
parser.add_option('--download', dest='is_download', action='store_true', help='download dojinshi or not')
parser.add_option('--id', type='int', dest='id', action='store', help='dojinshi id of nhentai')
parser.add_option('--ids', type='str', dest='ids', action='store', help='dojinshi id set, e.g. 1,2,3')
parser.add_option('--search', type='string', dest='keyword', action='store', help='keyword searched')
parser.add_option('--page', type='int', dest='page', action='store', default=1,
help='page number of search result')
parser.add_option('--path', type='string', dest='saved_path', action='store', default='',
help='path which save the dojinshi')
parser.add_option('--threads', '-t', type='int', dest='threads', action='store', default=1,
help='thread count of download dojinshi')
parser.add_option('--timeout', type='int', dest='timeout', action='store', default=30,
help='timeout of download dojinshi')
args, _ = parser.parse_args()
if args.ids:
_ = map(lambda id: id.strip(), args.ids.split(','))
args.ids = set(map(int, filter(lambda id: id.isdigit(), _)))
if args.is_download and not args.id and not args.ids and not args.keyword:
logger.critical('Dojinshi id/ids is required for downloading')
parser.print_help()
raise SystemExit
if args.id:
args.ids = (args.id, ) if not args.ids else args.ids
if not args.keyword and not args.ids:
parser.print_help()
raise SystemExit
if args.threads <= 0:
args.threads = 1
elif args.threads > 10:
logger.critical('Maximum number of used threads is 10')
raise SystemExit
return args
def _download(self, url, folder='', filename='', retried=False):
logger.info('Start downloading: {} ...'.format(url))
filename = filename if filename else os.path.basename(urlparse(url).path)
try:
with open(os.path.join(folder, filename), "wb") as f:
response = request('get', url, stream=True, timeout=self.timeout)
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(2048):
f.write(chunk)
except requests.HTTPError as e:
if not retried:
logger.error('Error: {}, retrying'.format(str(e)))
return self._download(url=url, folder=folder, filename=filename, retried=True)
else:
return None
except Exception as e:
logger.critical(str(e))
return None
return url
def download(self, queue, folder=''):
if not isinstance(folder, (str, unicode)):
folder = str(folder)
if self.path:
folder = os.path.join(self.path, folder)
if not os.path.exists(folder):
logger.warn('Path \'{}\' not exist.'.format(folder))
try:
os.makedirs(folder)
except EnvironmentError as e:
logger.critical('Error: {}'.format(str(e)))
raise SystemExit
else:
logger.warn('Path \'{}\' already exist.'.format(folder))
queue = [([url], {'folder': folder}) for url in queue]
self.thread_pool = threadpool.ThreadPool(self.thread_count)
requests_ = threadpool.makeRequests(self._download, queue, self._download_callback)
[self.thread_pool.putRequest(req) for req in requests_]
self.thread_pool.wait()
def finishLoadingEditor(self, original):
logger.debug('DocumentEditor finishLoadingEditor')
# execute original finishLoadingEditor()
original(self)
try:
# if toggle key is active, temporarily switch the active state
is_active = self.app.toggle_key_active ^ self.app.is_active
# check if we can proceed
if not is_active:
logger.debug("MailTrack is not active, so no MailTracking for you!")
return
# grab composeView instance (this is the WebView which contains the
# message editor) and check for the right conditions
try:
view = objc.getInstanceVariable(self, 'composeWebView')
except:
# was renamed in Lion
view = objc.getInstanceVariable(self, '_composeWebView')
# grab some other variables we need to perform our business
backend = self.backEnd()
htmldom = view.mainFrame().DOMDocument()
htmlroot = htmldom.documentElement()
messageType = self.messageType()
# XXX: hack alert! if message type is DRAFT, but we can determine this
# is actually a Send Again action, adjust the message type.
origmsg = backend.originalMessage()
if origmsg and messageType == DRAFT:
# get the message viewer for this message
viewer = MessageViewer.existingViewerShowingMessage_(origmsg)
if not viewer:
# XXX: this happens with conversation view active, not sure if this is stable enough though
messageType = SENDAGAIN
elif viewer:
# get the mailbox for the viewer
mailboxes = viewer.selectedMailboxes()
# get the Drafts mailbox
draftmailbox = viewer.draftsMailbox()
# check if they're the same; if not, it's a Send-Again
if draftmailbox not in mailboxes:
messageType = SENDAGAIN
# send original HTML to menu for debugging
self.app.html = htmlroot.innerHTML()
if not self.app.is_mailtracking:
logger.debug('mailtracking turned off in preferences, skipping that part')
elif messageType not in self.app.message_types_to_track:
logger.debug('message type "%s" not in %s, not tracking' % (
messageType,
self.app.message_types_to_track
))
else:
# move cursor to end of document
view.moveToEndOfDocument_(self)
# perform some general cleanups
logger.debug('calling cleanup_layout()')
if self.cleanup_layout(htmlroot, backend):
backend.setHasChanges_(False)
# move cursor to end of document
if self.app.move_cursor_to_top:
view.moveToBeginningOfDocument_(self)
# move to beginning of line
logger.debug('calling view.moveToBeginningOfLine()')
view.moveToBeginningOfLine_(self)
# done
logger.debug('MailTracking done')
except Exception:
logger.critical(traceback.format_exc())
if self.app.is_debugging:
NSRunAlertPanel(
'MailTrack caught an exception',
'The MailTrack plug-in caught an exception:\n\n' +
traceback.format_exc() +
'\nPlease contact the developer quoting the contents of this alert.',
None, None, None
)
if __name__ == '__main__':
'''
Note that we *must always* exit with 0. If we don't, the email we're
processing will be put back into the Postfix deferred queue and will get
processed again later. This will either end up in an infinite backlog of
email, or in responses to the same request being sent over and over.
'''
try:
starttime = time.time()
email_string = sys.stdin.read()
if not email_string:
logger.critical('error: no stdin')
sys.exit(0)
requested_addr = process_input(email_string)
if not requested_addr:
sys.exit(0)
except UnicodeDecodeError as ex:
# Bad input. Just log and exit.
logger.critical('error: UnicodeDecodeError')
except Exception as ex:
logger.critical('exception: %s: %s', ex, traceback.format_exc())
# Should we write this exception-causing email to disk?
if settings.EXCEPTION_DIR and email_string:
def main():
try:
_os.chdir(cachedir)
except EnvironmentError as e:
logger.critical('Cannot enter {} ({})'.format(e.filename, e.strerror))
_sys.exit(1)
if not _os.path.isdir(targetdir):
logger.critical('Cannot find {} (check targetdir variable)'.format(
targetdir))
_sys.exit(1)
try:
cacheini = open('cache.ini', 'r')
except EnvironmentError as e:
logger.critical('Cannot open {} ({})'.format(e.filename, e.strerror))
_sys.exit(1)
else:
with cacheini:
print('{}=== PREVIEW ==={reset}'.format(clicode.head1,
reset=clicode.reset))
movelist, dontmovelist = [], []
for line in cacheini:
dontmove = False
reline = _re.match(
'^([0-9A-Z]{32}-[0-9]+)(?:\=)(.+)(\.\w{1,3})(?:\n)$',
line)
if reline:
utfile = CacheFile(reline)
try:
utfile.setpath()
except CustomError as e:
logger.warning('{} extension has not been '
'recognized, {} will be left in the '
'cache'.format(utfile.realext,
utfile.cachename))
dontmove = True
else:
if not _os.path.isfile(utfile.cachename):
logger.warning('{} does not exist in the cache, '
'its line will be left in cache.ini, '
'but you should probably delete it '
'manually'.format(utfile.cachename))
dontmove = True
elif _os.path.isfile(_os.path.join(targetdir,
utfile.realpath,
utfile.realname +
utfile.realext)):
logger.warning('{} already exists, {} will be '
'left in the cache'.format(
_os.path.join(targetdir,
utfile.realpath,
utfile.realname +
utfile.realext),
utfile.cachename))
dontmove = True
else:
if not _re.match('^(\[Cache\]|\n)', line):
line = line.rstrip()
logger.warning('"{}" cannot be recognized, it will be '
'left in cache.ini'.format(line))
dontmove = True
if dontmove:
if not _re.match('^\n', line):
dontmovelist.append(line)
else:
movelist.append((utfile.cachename,
_os.path.join(targetdir, utfile.realpath),
_os.path.join(targetdir, utfile.realpath,
utfile.realname + utfile.realext), line))
print(utfile.cachename, _os.path.join(targetdir,
utfile.realpath, utfile.realname + utfile.realext),
sep=' {}-->{reset} '.format(clicode.arrow,
reset=clicode.reset))
if len(movelist) == 0:
logger.info('There are no files to move')
_sys.exit() # If writing a message here, the return status would be 1
question = inputtemplate.InputTemplate(
prompt='{}Do you want to move the file{P0s}? '
'[y|n]{reset} '.format(clicode.question,
reset=clicode.reset, **plural.set((len(movelist),))),
inputs={
'yes': ('y', 'yes'),
'no': ('n', 'no')
},
auto='yes',
wrong='Invalid input, please try again'
)
logger.debug('Do you want to move the file(s)? {}'.format(question.string))
if question.group == 'no':
logger.info('No changes were made')
_sys.exit() # If writing a message here, the return status would be 1
elif question.group == 'yes':
try:
if _os.path.isfile('cache.ini.tmp'):
logger.warning('Overwriting existing cache.ini.tmp')
open('cache.ini.tmp', 'w').close()
ftmp = open('cache.ini.tmp', 'a')
except EnvironmentError as e:
logger.critical('Cannot open {} ({})'.format(e.filename,
e.strerror))
_sys.exit(1)
else:
with ftmp:
moves = 0
errors = 0
for cache_file in movelist:
try:
if not _os.path.isdir(cache_file[1]):
_os.mkdir(cache_file[1])
_os.chmod(cache_file[1], 0o755)
logger.debug('{} directory created'.format(
cache_file[1]))
except EnvironmentError as e:
logger.error('Cannot create {} directory '
'({})'.format(e.filename, e.strerror))
dontmovelist.append(cache_file[3])
errors += 1
else:
try:
_shutil.move(cache_file[0], cache_file[2])
except EnvironmentError as e:
logger.error('Cannot move {} to {} ({})'.format(
cache_file[0], cache_file[2], e.strerror))
dontmovelist.append(cache_file[3])
errors += 1
else:
logger.debug('{} moved to {}'.format(cache_file[0],
cache_file[2]))
moves += 1
for line in dontmovelist:
ftmp.write(line)
filesmoved = '{} file{P0s} moved'.format(moves, **plural.set((moves,)))
if errors > 0:
filesmoved += ' ({} {}ERROR{reset}{P0s} reported)'.format(errors,
clicode.error, reset=clicode.reset,
**plural.set((errors,)))
logger.info(filesmoved)
if moves > 0:
bkpext = _time.strftime('%Y%m%d%H%M%S')
while _os.path.isfile('cache.ini.bak.' + bkpext):
bkpext = repr(int(bkpext) + 1)
try:
_shutil.copy('cache.ini', 'cache.ini.bak.' + bkpext)
except EnvironmentError:
logger.critical('Couldn\'t create a backup for cache.ini, '
'to complete the operations you have to '
'overwrite it manually with cache.ini.tmp '
'(which is the updated version)')
raise
else:
logger.info('cache.ini backup successfully created')
cachelist = _os.listdir('.')
datelist = []
for f in cachelist:
redate = _re.match('^(?:cache\.ini\.bak\.)([0-9]{14})$', f)
if redate:
datelist.append(redate.group(1))
datelist.sort(reverse=True)
if backupsN >= 0:
for d in datelist[backupsN:]:
try:
_os.remove('cache.ini.bak.' + d)
except EnvironmentError as e:
logger.error('Couldn\'t delete obsolete '
'backup: {} ({})'.format(e.filename,
e.strerror))
try:
_shutil.move('cache.ini.tmp', 'cache.ini')
except EnvironmentError:
logger.critical('Couldn\' t overwrite cache.ini (the '
'old version) with cache.ini.tmp (the '
'updated version), please do it manually')
raise
else:
logger.info('cache.ini correctly updated')
else:
try:
_os.remove('cache.ini.tmp')
except EnvironmentError as e:
logger.error('Couldn\'t delete {} ({})'.format(e.filename,
e.strerror))
def get_seeds(self):
'''download and store seeds'''
session = FuturesSession()
def get_req(future):
response = future.result()
if response.status_code not in range(400,520):
if "text/html" in response.headers['content-type']:
article = self.extract(response, depth=0, filters=False)
if article["status"]:
outlinks = article["outlinks"]
del article["outlinks"]
try:
self.db.data.insert(article)
try:
ex = [n["_id"] for n in self.db.seeds.find({"url":article["url"]},{"_id":1})]
self.db.seeds.find_one_and_update({"url":article["url"]}, self.show(article))
except Exception as e:
pass
#self.db.seeds.update(article)
for n in outlinks:
#bulk = [InsertOne(x for x in outlinks if x["url"] not in self.db.data.distinct("url") and x["url"] not in self.db.queue.distinct("url"))]
if self.db.queue.count({"url":n["url"]}) > 0:
pass
if self.db.data.count({"url":n["url"]}) > 0:
pass
else:
try:
self.db.queue.insert_one(n)
except pymongo.errors.DuplicateKeyError:
#print "Already in queue"
pass
return True
except pymongo.errors.DuplicateKeyError:
#~ print "Article is already in DB outlinks not put in Queue"
#~ try:
#~ self.db.seeds.find_one_and_update({"url":article["url"]}, self.show(article))
#~ except Exception as e:
#~ print e, article
#~ pass
#~ return True
pass
else:
try:
self.db.seeds.find_one_and_update({"url":article["url"]}, self.show(article))
except Exception as e:
pass
return False
else:
status_code = 406
msg = "Format de la page non supporté: %s" %response.headers['content-type']
self.db.seeds.find_one_and_update({"url":response.url}, {"$set":{"status": False, "status_code": status_code, "msg": msg}})
try:
self.db.logs.insert({"$set":{"url":response.url, "status": False, "status_code": status_code, "msg": msg}})
except pymongo.errors.DuplicateKeyError:
pass
return False
else:
status_code = response.status_code
msg = "Page indisponible"
self.db.seeds.find_one_and_update({"url":response.url}, {"$set":{"status": False, "status_code": status_code, "msg": msg}})
try:
self.db.logs.insert({"url":response.url, "status": False, "status_code": status_code, "msg": msg})
except pymongo.errors.DuplicateKeyError:
pass
return False
for x in self.db.seeds.find({},{"url":1, "depth":1, "_id":0,"status":1}):
url = Url(x["url"])
#headers = {"User-agent": "Mozilla/5.0","Connection": "close"}
future = session.get(url.url, verify=False)
depth = 0
try:
future.add_done_callback(get_req)
except Exception as e:
logger.critical(e)
return bool(self.db.queue.count() > 0)
def fix(self):
try:
# if toggle key is active, temporarily switch the active state
is_active = self.app.toggle_key_active ^ self.app.is_active
# check if we can proceed
if not is_active:
logger.debug("QuoteFix is not active, so no QuoteFixing for you!")
return
# Grab some variables we need to perform our business
view = self.composeWebView() # contains the message editor
backend = self.backEnd()
htmldom = view.mainFrame().DOMDocument()
htmlroot = htmldom.documentElement()
messageType = self.messageType()
# XXX: hack alert! if message type is DRAFT, but we can determine this
# is actually a Send Again action, adjust the message type.
origmsg = backend.originalMessage()
if origmsg and messageType == DRAFT and origmsg.type() == 0:
messageType = SENDAGAIN
# send original HTML to menu for debugging
self.app.html = htmlroot.innerHTML()
# provide custom attribution?
attributor = None
if self.app.use_custom_reply_attribution and messageType in [ REPLY, REPLY_ALL, REPLY_AS ]:
logger.debug("calling customize_attribution() for reply{-all,-as}")
attributor = CustomizedAttribution.customize_reply
elif self.app.use_custom_sendagain_attribution and messageType in [ SENDAGAIN ]:
logger.debug("calling customize_attribution() for Send Again")
attributor = CustomizedAttribution.customize_sendagain
elif self.app.use_custom_forwarding_attribution and messageType == FORWARD:
logger.debug("calling customize_attribution() for forwarding")
attributor = CustomizedAttribution.customize_forward
if attributor:
# play nice with Attachment Tamer
try:
message = backend.draftMessage()
except:
try:
copy = backend.copyOfContentsForDraft_shouldBePlainText_isOkayToForceRichText_(True, False, True)
except:
# Yosemite
copy = backend.copyOfContentsForDraft_shouldBePlainText_isOkayToForceRichText_isMailDropPlaceholderMessage_(True, False, True, False)
message = backend._makeMessageWithContents_isDraft_shouldSign_shouldEncrypt_shouldSkipSignature_shouldBePlainText_(
copy,
True,
False,
False,
False,
False
)
try:
for original in objc.getInstanceVariable(backend, '_originalMessages'):
attributor(
app = self.app,
editor = self,
dom = htmldom,
reply = message,
inreplyto = original,
)
backend.setHasChanges_(False)
except:
# ignore when not debugging
if self.app.is_debugging:
raise
# should we be quotefixing?
if not self.app.is_quotefixing:
logger.debug('quotefixing turned off in preferences, skipping that part')
elif messageType not in self.app.message_types_to_quotefix:
logger.debug('message type "%s" not in %s, not quotefixing' % (
messageType,
self.app.message_types_to_quotefix
))
else:
# remove attachment placeholders?
if self.app.remove_attachment_placeholders:
self.remove_attachment_placeholders(backend, htmlroot)
# move cursor to end of document
view.moveToEndOfDocument_(self)
# remove quotes?
if self.app.remove_quotes:
logger.debug('calling remove_quotes()')
self.remove_quotes(htmldom, self.app.remove_quotes_level)
# make quotes selectable?
if self.app.selectable_quotes:
logger.debug('calling make_selectable_quotes()')
self.make_selectable_quotes(view, htmldom)
# remove signature from sender
if not self.app.keep_sender_signature:
logger.debug('calling remove_old_signature()')
self.remove_old_signature(htmldom, view)
# place cursor above own signature (if any)
logger.debug('calling move_above_new_signature()')
if self.move_above_new_signature(htmldom, view):
# insert a paragraph break?
if not self.app.no_whitespace_below_quote:
view.insertParagraphSeparator_(self)
else:
view.insertNewline_(self)
# perform some general cleanups
logger.debug('calling cleanup_layout()')
self.cleanup_layout(htmlroot, backend)
# move cursor to top of document
if self.app.move_cursor_to_top:
view.moveToBeginningOfDocument_(self)
# move to beginning of line
logger.debug('calling view.moveToBeginningOfLine()')
view.moveToBeginningOfLine_(self)
# done
logger.debug('QuoteFixing done')
except Exception:
logger.critical(traceback.format_exc())
if self.app.is_debugging:
NSRunAlertPanel(
'QuoteFix caught an exception',
'The QuoteFix plug-in caught an exception:\n\n' +
traceback.format_exc() +
'\nPlease contact the developer quoting the contents of this alert.',
None, None, None
)
def finishLoadingEditor(self, original):
logger.debug('DocumentEditor finishLoadingEditor')
# execute original finishLoadingEditor()
original(self)
try:
# if toggle key is active, temporarily switch the active state
is_active = self.app.toggle_key_active ^ self.app.is_active
# check if we can proceed
if not is_active:
logger.debug("QuoteFix is not active, so no QuoteFixing for you!")
return
# grab composeView instance (this is the WebView which contains the
# message editor) and check for the right conditions
try:
view = objc.getInstanceVariable(self, 'composeWebView')
except:
# was renamed in Lion
view = objc.getInstanceVariable(self, '_composeWebView')
# grab some other variables we need to perform our business
backend = self.backEnd()
htmldom = view.mainFrame().DOMDocument()
htmlroot = htmldom.documentElement()
messageType = self.messageType()
# XXX: hack alert! if message type is DRAFT, but we can determine this
# is actually a Send Again action, adjust the message type.
origmsg = backend.originalMessage()
if origmsg and messageType == DRAFT:
# get the message viewer for this message
viewer = MessageViewer.existingViewerShowingMessage_(origmsg)
if not viewer:
# XXX: this happens with conversation view active, not sure if this is stable enough though
messageType = SENDAGAIN
elif viewer:
# get the mailbox for the viewer
mailboxes = viewer.selectedMailboxes()
# get the Drafts mailbox
draftmailbox = viewer.draftsMailbox()
# check if they're the same; if not, it's a Send-Again
if draftmailbox not in mailboxes:
messageType = SENDAGAIN
# send original HTML to menu for debugging
self.app.html = htmlroot.innerHTML()
# should we be quotefixing?
if not self.app.is_quotefixing:
logger.debug('quotefixing turned off in preferences, skipping that part')
elif messageType not in self.app.message_types_to_quotefix:
logger.debug('message type "%s" not in %s, not quotefixing' % (
messageType,
self.app.message_types_to_quotefix
))
else:
# remove attachment placeholders?
if self.app.remove_attachment_placeholders:
self.remove_attachment_placeholders(backend, htmlroot)
backend.setHasChanges_(False)
# move cursor to end of document
view.moveToEndOfDocument_(self)
# remove quotes?
if self.app.remove_quotes:
logger.debug('calling remove_quotes()')
self.remove_quotes(htmldom, self.app.remove_quotes_level)
backend.setHasChanges_(False)
# make quotes selectable?
if self.app.selectable_quotes:
logger.debug('calling make_selectable_quotes()')
self.make_selectable_quotes(view, htmldom)
backend.setHasChanges_(False)
# remove signature from sender
if not self.app.keep_sender_signature:
logger.debug('calling remove_old_signature()')
if self.remove_old_signature(htmldom, view):
backend.setHasChanges_(False)
# place cursor above own signature (if any)
logger.debug('calling move_above_new_signature()')
if self.move_above_new_signature(htmldom, view):
backend.setHasChanges_(False)
else:
view.insertNewline_(self)
# perform some general cleanups
logger.debug('calling cleanup_layout()')
if self.cleanup_layout(htmlroot, backend):
backend.setHasChanges_(False)
# move cursor to end of document
if self.app.move_cursor_to_top:
view.moveToBeginningOfDocument_(self)
# provide custom attribution?
attributor = None
if self.app.use_custom_reply_attribution and messageType in [ REPLY, REPLY_ALL, REPLY_AS ]:
logger.debug("calling customize_attribution() for reply{-all,-as}")
attributor = CustomizedAttribution.customize_reply
elif self.app.use_custom_sendagain_attribution and messageType in [ SENDAGAIN ]:
logger.debug("calling customize_attribution() for Send Again")
attributor = CustomizedAttribution.customize_sendagain
elif self.app.use_custom_forwarding_attribution and messageType == FORWARD:
logger.debug("calling customize_attribution() for forwarding")
attributor = CustomizedAttribution.customize_forward
if attributor:
# play nice with Attachment Tamer
try:
message = backend.draftMessage()
except:
message = backend._makeMessageWithContents_isDraft_shouldSign_shouldEncrypt_shouldSkipSignature_shouldBePlainText_(
backend.copyOfContentsForDraft_shouldBePlainText_isOkayToForceRichText_(True, False, True),
True,
False,
False,
False,
False
)
try:
for original in objc.getInstanceVariable(backend, '_originalMessages'):
attributor(
app = self.app,
editor = self,
dom = htmldom,
reply = message,
inreplyto = original,
)
backend.setHasChanges_(False)
except:
# ignore when not debugging
if self.app.is_debugging:
raise
# move to beginning of line
logger.debug('calling view.moveToBeginningOfLine()')
view.moveToBeginningOfLine_(self)
# done
logger.debug('QuoteFixing done')
except Exception:
logger.critical(traceback.format_exc())
if self.app.is_debugging:
NSRunAlertPanel(
'QuoteFix caught an exception',
'The QuoteFix plug-in caught an exception:\n\n' +
traceback.format_exc() +
'\nPlease contact the developer quoting the contents of this alert.',
None, None, None
)
def process_email(self, email_string):
'''
Processes the given email and sends a response.
Returns True if successful, False or exception otherwise.
'''
self._email_string = email_string
if not self._parse_email(email_string):
return False
# Look up all config entries matching the requested address.
request_conf = [item for item in self._conf if item['email_addr'] == self.requested_addr]
# If we didn't find anything for that address, exit.
if not request_conf:
logger.info('fail: invalid requested address: %s', self.requested_addr)
return False
# Check if the user is (or should be) blacklisted
if not self._check_blacklist():
logger.info('fail: blacklist')
return False
# Process each config entry found the for the requested address separately.
# Don't fail out early, since the other email send method has a chance
# to succeed even if one fails. (I.e., SMTP will succeed even if there's
# a SES service problem.)
full_success = True
exception_to_raise = None
for conf in request_conf:
attachments = None
if conf['attachments']:
attachments = []
for attachment_info in conf['attachments']:
bucketname, bucket_filename, attachment_filename = attachment_info
attachments.append((aws_helpers.get_s3_attachment(settings.ATTACHMENT_CACHE_DIR,
bucketname,
bucket_filename),
attachment_filename))
extra_headers = {'Reply-To': self.requested_addr}
if self._requester_msgid:
extra_headers['In-Reply-To'] = self._requester_msgid
extra_headers['References'] = self._requester_msgid
raw_response = sendmail.create_raw_email(self._requester_addr,
self._response_from_addr,
self._subject,
conf['body'],
attachments,
extra_headers)
if not raw_response:
full_success = False
continue
if conf.get('send_method', '').upper() == 'SES':
# If sending via SES, we'll use its DKIM facility -- so don't do it here.
try:
if not sendmail.send_raw_email_amazonses(raw_response,
self._response_from_addr):
return False
except BotoServerError as ex:
if ex.error_message == 'Address blacklisted.':
logger.critical('fail: requester address blacklisted by SES')
else:
exception_to_raise = ex
full_success = False
continue
else:
raw_response = _dkim_sign_email(raw_response)
if not sendmail.send_raw_email_smtp(raw_response,
settings.COMPLAINTS_ADDRESS, # will be Return-Path
self._requester_addr):
full_success = False
continue
if exception_to_raise:
raise exception_to_raise
return full_success
def _download_callback(self, request, result):
if not result:
logger.critical('Too many errors occurred, quit.')
raise SystemExit
logger.log(15, '{} download successfully'.format(result))
def get_config(self, timeout=10):
"""
Метод для получения конфигурационного файла целевого оборудования.
В методе определены oid'ы используемого оборудования.
Метод работает следующим образом - по snmp запрашивается oid_sysname
и по нему определяется тип оборудования, по типу оборудования
запрашиваются соответствующие oid и на указанный TFTP сервер
закачивается конфигурационный файл оборудования, далее он считывается
в виде строки и удаляется с сервера.
:param timeout: таймаут на получение конфигурационного файла
:rtype: строка с конфигурационным файлом оборудования
"""
try:
if not self.eqp_type:
self.get_eqp_type()
if not self.firmware:
self.get_firmware_version()
except DlinkInitException as dlink_exc:
logger.error(dlink_exc)
raise DlinkConfigException(
self.ip, 'дальнейшая работа с оборудованием невозможна'
)
cfg_file_name = 'config-%s.cfg' % self.ip
# набор oid'ов для конфигурации обрудования DES-3*** на отдачу
# конфигурационного файла на TFTP сервер
oids_des = (
('1.3.6.1.4.1.171.12.1.2.1.1.3.3', IpAddress(self.tftp_server)),
('1.3.6.1.4.1.171.12.1.2.1.1.4.3', Integer(2)),
('1.3.6.1.4.1.171.12.1.2.1.1.5.3', OctetString(cfg_file_name)),
('1.3.6.1.4.1.171.12.1.2.1.1.6.3', Integer(3)),
('1.3.6.1.4.1.171.12.1.2.1.1.7.3', Integer(2)),
('1.3.6.1.4.1.171.12.1.2.1.1.8.3', Integer(3))
)
# набор oid'ов для конфигурации обрудования DGS-3*** на отдачу
# конфигурационного файла на TFTP сервер
oids_dgs = (
('1.3.6.1.4.1.171.12.1.2.18.1.1.3.3', IpAddress(self.tftp_server)),
('1.3.6.1.4.1.171.12.1.2.18.1.1.5.3', OctetString(cfg_file_name)),
('1.3.6.1.4.1.171.12.1.2.18.1.1.8.3', Integer(2)),
('1.3.6.1.4.1.171.12.1.2.18.1.1.12.3', Integer(3))
)
# набор oid'ов для конфигурации обрудования DGS-3100 на отдачу
# конфигурационного файла на TFTP сервер
oids_tg = (
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.4.1', IpAddress(self.ip)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.5.1', Integer(1)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.6.1', OctetString('startupConfig')),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.7.1', Integer(3)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.8.1', Integer(3)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.9.1', IpAddress(self.tftp_server)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.10.1', Integer(1)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.11.1', OctetString(cfg_file_name)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.12.1', Integer(3)),
('1.3.6.1.4.1.171.10.94.89.89.87.2.1.17.1', Integer(4))
)
cfg_file_end = 'End of configuration file'
# TG
if 'DGS-3100' in self.eqp_type:
current_eqp = oids_tg
# переопределяем окончание конфигурационного файла для
# оборудования DGS-3100
cfg_file_end = '! VOICE VLAN'
# DES-3526, DES-3528, DES-3010G, DGS-3200
elif 'DES-3526' in self.eqp_type \
or 'DES-3528' in self.eqp_type \
or 'DES-3010G' in self.eqp_type \
or 'DGS-3200' in self.eqp_type:
current_eqp = oids_des
# DGS
elif 'DGS-3' in self.eqp_type:
current_eqp = oids_dgs
# DES
elif 'DES-3' in self.eqp_type:
if self.firmware:
if float(self.firmware[:4]) >= 4.00:
current_eqp = oids_dgs
else:
current_eqp = oids_des
else:
current_eqp = oids_des
else:
raise DlinkConfigException(
self.ip, 'не удалось определить нужный набор oid '
'для настройки оборудования на отдачу конфигурационного файла'
)
# получаем конфиг, если определить тип оборудование не получилось,
# то выводим соответствующее сообщение
try:
self.snmp.set(*current_eqp)
except snmp.SnmpSetTimeoutException as snmp_exc:
logger.critical(snmp_exc)
raise DlinkConfigException(
self.ip, 'не удалось настроить оборудование на отдачу '
'конфигурационного файла'
)
logger.debug(
'%s - оборудование настроено на отдачу конфигурационного файла '
'успешно' % self.ip
)
result = None
file_path = os.path.join(self.tftp_path, cfg_file_name)
if self.config_load_method == 'local':
open_func = open
rm_func = os.remove
conn_close_func = lambda: None
elif self.config_load_method == 'ssh':
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
try:
ssh.connect(
self.tftp_server,
username=self.username,
password=self.password
)
except socket.error as exc:
logger.error(
'%s - %s' % (self.ip, exc)
)
raise DlinkConfigException(
self.ip, 'не удалось подключиться к серверу %s'
'по протоколу ssh' % self.tftp_server
)
sftp = ssh.open_sftp()
open_func = sftp.open
rm_func = sftp.remove
conn_close_func = ssh.close
else:
raise DlinkConfigException(
self.ip, 'неверно указан метод загрузки конфигурационного файла'
)
_c = 0
while 1:
time.sleep(1)
if _c < timeout:
try:
_f = open_func(file_path, mode='r')
# обработка ситуации когда файл еще не создан
except IOError as io_exc:
_c += 1
else:
cfg_file = _f.read()
if cfg_file_end in cfg_file:
_f.close()
rm_func(file_path)
conn_close_func()
result = cfg_file.replace('\r\n', '\n')
break
else:
end_not_obtained = True
_c += 1
_f.close()
else:
conn_close_func()
if 'io_exc' in locals():
raise DlinkConfigException(
self.ip, 'конфигурационного файла %s не существует '
'на сервере %s' %
(file_path, self.tftp_server)
)
elif 'end_not_obtained' in locals():
raise DlinkConfigException(
self.ip, 'конец файла %s не получен за %s секунд' %
(file_path, timeout)
)
else:
raise DlinkConfigException(
self.ip, 'не удалось получить конфигурационный файл '
'%s с сервера %s по неизвестной причине' %
(file_path, self.tftp_server)
)
logger.info(
'%s - конфигурационный файла получен успешно' % self.ip
)
self.chassis.config_file = result
return result
