def main():
md = MetaDoc()
sd_all = [SimpleDoc(info) for info in md.get_meta_info(scope = 'simple')]
sd_list = [sd for sd in sd_all if list_check(sd)]
sd_error_list = [sd for sd in sd_all if not list_check(sd)]
for sd in sd_list:
try:
if sd.is_translated:
print sd.meta_info['file_id'], '/', sd.en_title, '/', sd.ja_title.encode('utf-8')
else:
print sd.meta_info['file_id'], '/', sd.en_title, '/', 'NOT_TRANSLATED'
except Exception, errormsg:
print errormsg, sd.meta_info['file_id']
def __init__(self):
self.base_template = Template(open(CONFIG['BASE_TAMPLATE'], 'r').read().decode('utf-8'))
self.chapter_template = Template(open(CONFIG['CHAPTER_TEMPLATE'], 'r').read().decode('utf-8'))
self.md = MetaDoc()
self.categories = {}
# Simple Documents
sd_all = [info for info in self.md.get_meta_info(scope = 'simple')]
sd_all = guess_categories(sd_all)
sd_all = [SimpleDoc(info) for info in sd_all]
sd_list = [sd for sd in sd_all if self.list_check(sd)]
sd_error_list = [sd for sd in sd_all if not self.list_check(sd)]
for sd in sd_list:
self.add_doc(sd)
hb_all = [info for info in self.md.get_meta_info(scope = 'handbook')]
hb_all = guess_categories_coverpage(hb_all)
hb_all = [Handbook(info) for info in hb_all]
for hb in hb_all:
self.add_doc(hb)
def send_element(element, conf, send_cache, dryrun, verbose, cache_only):
"""Attempts to gather and send information defined by element to server.
If send_cache is True or dryrun is False, cache will be ignored.
If cache_only is True, only cache is checked and no new data is gathered.
@param element: Element class that should be fetched.
@type element: L{MetaElement} sub class.
@param conf: Configuration dictionary.
@type conf: dict
@param send_cache: Indicates whether cache should be included.
@type send_cache: bool
@param dryrun: Indicates whether the script is doing a dry run.
@type dryrun: bool
@param verbose: Indicates whether or not the script runs in verbose mode.
@type verbose: bool
@param cache_only: Indicates whether to send only cache for this element.
@type cache_only: bool
"""
if dryrun and cache_only:
# We're doing a dry run and we've reached the cached items
return
m = MetaDoc(conf.get("site_name"))
element_processor = get_element_processor(element, send_cache,
verbose, dryrun)
if not cache_only:
# If we're doing cache only, we've reached the possible_send_elements
# loop and do not want or need to remove elements from it anymore.
site_element = element.site_handler()
site_element.populate()
element_processor.add_elements(site_element.fetch())
m.reg_meta_element(element_processor)
# Build URL
url = "%s%s" % (conf['host'], element.url)
if bool_conf_value(conf.get('trailing_slash',"")):
url = "%s/" % url
# Check to see if there is any content to transfer in the MetaDoc.
if m.has_content():
if verbose:
print "-" * 70
print "Connecting to host: %s" % url
print "Using key: %s" % conf['key']
print "Using certificate: %s" % conf['cert']
print "-" * 70
print "Sent data:\n%s" % ("-" * 70)
print m.get_xml(pretty=True)
if not dryrun:
server_response = utils.send_document(url,
conf['key'],
conf['cert'],
m)
if server_response is False:
if not send_cache:
logging.warning("Could not send data to server, "
"but running with --no-cache, so data was not cached.")
else:
Cacher(element.xml_tag_name, m)
else:
if verbose:
print "%s\nRecieved data:\n%s" % ("-" * 70, "-" * 70)
print server_response
print "-" * 70
utils.check_response(element.xml_tag_name,
m,
server_response,
send_cache)
else:
if verbose and not cache_only:
print "No data to send for \"%s\"." % element.xml_tag_name
logging.info(("No data to send for \"%s\".") % element.xml_tag_name)
#!env python
from simple_doc import SimpleDoc
from handbook import Handbook
from metadoc import MetaDoc
m = MetaDoc()
for info in [i for i in m.get_meta_info(scope = 'handbook')]:
doc = Handbook(info)
if doc.has_child():
for c in doc.childs:
print "%s -> %s" % (c.meta_info['parent'], c.meta_info['file_en_path'])
def send_element(element, conf, send_cache, dryrun, verbose, cache_only):
"""Attempts to gather and send information defined by element to server.
If send_cache or dryrun is False, cache will be ignored.
If cache_only is True, only cache is checked and no new data is gathered.
@param element: Element class that should be fetched.
@type element: L{MetaElement} sub class.
@param conf: Configuration dictionary.
@type conf: dict
@param send_cache: Indicates whether cache should be included.
@type send_cache: bool
@param dryrun: Indicates whether the script is doing a dry run.
@type dryrun: bool
@param verbose: Indicates whether or not the script runs in verbose mode.
@type verbose: bool
@param cache_only: Indicates whether to send only cache for this element.
@type cache_only: bool
"""
if dryrun and cache_only:
# We're doing a dry run and we've reached the cached items
return
m = MetaDoc(conf.get("site_name"))
element_processor = get_element_processor(element, send_cache,
verbose, dryrun)
if not cache_only:
# If we're doing cache only, we've reached the possible_send_elements
# loop and do not want or need to remove elements from it anymore.
site_element = element.site_handler()
site_element.populate()
element_processor.add_elements(site_element.fetch())
m.reg_meta_element(element_processor)
# Build URL
url = "%s%s" % (conf['host'], element.url)
if conf.get('trailing_slash',"").lower() == 'true'\
or conf.get('trailing_slash',"").lower() == 'yes':
url = "%s/" % url
# Check to see if there is any content to transfer in the MetaDoc.
if m.has_content():
# Do not want to send any data if we're doing a dry run.
if not dryrun:
cli = metahttp.XMLClient(url, conf['key'], conf['cert'])
if verbose:
print "-" * 70
print "Connecting to host: %s" % url
print "Using key: %s" % conf['key']
print "Using certificate: %s" % conf['cert']
print "-" * 70
print "Sent data:\n%s" % ("-" * 70)
print m.get_xml(pretty=True)
# Will not recieve any data on a dry run.
if not dryrun:
try:
res = cli.send(m.get_xml())
except (urllib2.HTTPError, urllib2.URLError) as httperror:
logging.error(("Unable to connect to server address \"%s\". "
"Error: %s") % (url, httperror))
# Since we're unable to send the document to the server, we will
# cache it so that we can send it at a later date.
if not send_cache:
logging.warning("Could not send data to server, "
"but running with --no-cache, so data was not cached.")
else:
Cacher(element.xml_tag_name, m)
else:
if res:
xml_data = res.read()
if verbose:
print "%s\nRecieved data:\n%s" % ("-" * 70, "-" * 70)
print xml_data
print "-" * 70
utils.check_response(element.xml_tag_name,
m,
xml_data,
send_cache)
else:
logging.error(("Server returned an empty response when "
"attempting to send \"%s\". Caching data.") %
element.xml_tag_name)
# Since we recieved an empty response from the server we have
# not recieved any reciept for any elements and must cache
# them.
if not send_cache:
logging.warning("No data returned from server, but "
"running with --no-cache, so data was not cached.")
else:
Cacher(element.xml_tag_name, m)
else:
if verbose and not cache_only:
print "No data to send for \"%s\"." % element.xml_tag_name
logging.info(("No data to send for \"%s\".") % element.xml_tag_name)
sys.exit(2)
else:
if event_type == "resourceDown":
if reason is None:
print "Recieved resource down handle, but missing reason."
print __doc__
sys.exit(2)
if share_down is None:
print "Recieved resource down handle, but missing share down."
print __doc__
sys.exit(2)
if date_down is None:
print "Recieved resource down handle, but missing date down."
# We have everything we require to create an event.
# Attempt to find already cached data:
m = MetaDoc(site_name)
c = Cacher("events")
cached_data = c.get_cache()
if cached_data is not None:
processor = Events.from_xml_element(cached_data, Events)
if processor is None:
print "Found previous event cache, but could not load. Please check "
print "\"%s\" for errors. " % c.file_path
print "Halting."
sys.exit(2)
else:
c.remove_cache()
else:
processor = Events()
if event_type == "resourceUp":
e = ResourceUpEntry(date_up, reason, remarks)
class TransStatus(object):
def __init__(self):
self.base_template = Template(open(CONFIG['BASE_TAMPLATE'], 'r').read().decode('utf-8'))
self.chapter_template = Template(open(CONFIG['CHAPTER_TEMPLATE'], 'r').read().decode('utf-8'))
self.md = MetaDoc()
self.categories = {}
# Simple Documents
sd_all = [info for info in self.md.get_meta_info(scope = 'simple')]
sd_all = guess_categories(sd_all)
sd_all = [SimpleDoc(info) for info in sd_all]
sd_list = [sd for sd in sd_all if self.list_check(sd)]
sd_error_list = [sd for sd in sd_all if not self.list_check(sd)]
for sd in sd_list:
self.add_doc(sd)
hb_all = [info for info in self.md.get_meta_info(scope = 'handbook')]
hb_all = guess_categories_coverpage(hb_all)
hb_all = [Handbook(info) for info in hb_all]
for hb in hb_all:
self.add_doc(hb)
def add_doc(self, doc):
for (k, v) in doc.meta_info['ja_memberof'].items():
if self.categories.has_key(k):
self.categories[k]['member'].append(doc)
else:
parent = self.md.get_parent_category_title(lang='ja', c_id=k)
self.categories.update({k: {'member': [], 'title': v, 'parent': parent}})
self.categories[k]['member'].append(doc)
def list_check(self, sd):
if sd.meta_info_error:
return False
if sd.meta_info['file_id'] in CONFIG['NON_LIST_FILE']:
return False
if not sd.meta_info['en_memberof']:
return False
return True
def dump(self):
chapters = []
for category in sorted(self.categories.items()):
records = []
for record in self.record(category[0]):
records.append(record)
if len(records) > 0:
c_title = u"%s -> %s" % (category[1]['parent'], category[1]['title'])
chapters.append(self.chapter_template.substitute(chapter = c_title, records = u"".join(records)))
d = u"%s" % date.today().strftime('%d %b %Y')
print self.base_template.substitute(date = d, chapters = u"".join(chapters)).encode('utf-8')
def record(self, category_id):
docs = self.categories[category_id]['member']
for doc in docs:
attrs = {
'en_url': escape(doc_url(doc, lang='en')),
'en_cvs': escape(doc_url(doc, lang='en', cvs=True)),
'en_cvs_rev': doc.en_cvs_rev,
'en_title': doc.en_title,
}
if doc.is_translated:
if doc.en_cvs_rev == doc.ja_org_rev:
template = Template(open(CONFIG['RECORD_LATEST_TEMPLATE'], 'r').read().decode('utf-8'))
else:
template = Template(open(CONFIG['RECORD_TEMPLATE'], 'r').read().decode('utf-8'))
doc_diff_url = docdiff_url(doc)
if doc_diff_url == None:
doc_diff_url = ""
else:
doc_diff_url = CONFIG['DIFF_URL'] + doc_diff_url
try:
translator = doc.translator
except:
translator = ""
attrs.update({
'ja_url': escape(doc_url(doc, lang='ja')),
'ja_cvs': escape(doc_url(doc, lang='ja', cvs=True)),
'ja_cvs_rev': doc.ja_org_rev,
'diff_url': escape(doc_url(doc, diff=True)),
'doc_diff_url': escape(doc_diff_url),
'translator': escape(translator)
})
else:
template = Template(open(CONFIG['RECORD_NOT_TRANSLATE_TEMPLATE'], 'r').read().decode('utf-8'))
yield template.substitute(attrs)