def main():
ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH)
index = '<html><head></head><body>'
index += INDEX_HTML_HEADER + '<br/> <br/>'
index += '<a href="' + LOCAL_URL_PREFIX + 'master.zip">master.zip</a><br/>'
index += '<a href="' + LOCAL_URL_PREFIX + 'ckan.exe">ckan.exe</a> | <a href="' + LOCAL_URL_PREFIX + 'netkan.exe">netkan.exe</a><br/>'
index += '<a href="' + LOCAL_URL_PREFIX + 'log.txt">MirrorKAN log</a><br/> <br/>'
info = log.getInfo()
index += '<font style="color: #669900; font-weight: normal;">Info:<br/>'
for item in info:
index += item + '<br/>'
index += '</font><br/>'
warnings = log.getWarnings()
index += '<font style="color: #CC3300; font-weight: normal;">Warnings:<br/>'
for item in warnings:
index += item + '<br/>'
index += '</font><br/>'
errors = log.getErrors()
index += '<font style="color: #CC3300; font-weight: bold;">Errors:<br/>'
for item in errors:
index += item + '<br/>'
index += '</font><br/><br/>'
index += 'Modules list:<br/>'
for ckan_module in ckan_json:
identifier = ckan_module[0]['identifier']
print 'Generating info for module %s' % identifier
version = ckan_module[0]['version']
filename = ckan_module[0]['x_mirrorkan_cached_filename']
file_status = ckan_module[0]['x_mirrorkan_download_status']
style = "color: #339900;"
if file_status is DLRESULT_HTTP_ERROR_NOT_CACHED:
style = "color: #CC3300; font-weight: bold;"
elif file_status is DLRESULT_HTTP_ERROR_CACHED:
style = "color: #FFD700; font-weight: bold;"
index += '<font style="' + style + '">'
index += ' ' + identifier + ' - ' + version + ' - '
index += 'Status: ' + ckan_module[0]['x_mirrorkan_status'] + '(' + str(ckan_module[0]['x_mirrorkan_download_status']) + ') - '
index += 'Last update: ' + ckan_module[0]['x_last_updated'] + '<br/>'
index += '</font>'
index += '</body></html>'
print 'Writing index.html'
index_file = open(os.path.join(FILE_MIRROR_PATH, 'index.html'), 'w')
index_file.write(index)
index_file.close()
def main():
print 'Building CKAN RSS Feed..'
ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH)
unsorted_feed_path = os.path.join(FEED_PATH, 'unsorted.json')
with open(unsorted_feed_path, 'w') as unsorted_feed_file:
print 'Writing %s' % unsorted_feed_path
json.dump(ckan_json, unsorted_feed_file, indent=4, sort_keys=True)
sorted_ckan_json = sorted(ckan_json, key=lambda ckan: ckan[0].get('x_last_updated_ts'))
sorted_feed_path = os.path.join(FEED_PATH, 'sorted.json')
with open(sorted_feed_path, 'w') as sorted_feed_file:
print 'Writing %s' % sorted_feed_path
json.dump(sorted_ckan_json, sorted_feed_file, indent=4, sort_keys=True)
rssitems = []
module_number = 1000000
for ckan_module in sorted_ckan_json:
module_number = module_number + 1
# Fallback for link in case nothing can be determinded
link = 'http://kerbalstuff.com/' + str(module_number)
title = ckan_module[0]['name']
if 'resources' in ckan_module[0]:
if 'kerbalstuff' in ckan_module[0]['resources']:
link = ckan_module[0]['resources']['kerbalstuff']
# elif 'homepage' in ckan_module[0]['resources']:
# link = ckan_module[0]['resources']['homepage']
elif 'repository' in ckan_module[0]['resources']:
link = ckan_module[0]['resources']['repository']
# Make links unique
link = link + '#' + str(module_number)
description = ckan_module[0]['abstract']
guid = PyRSS2Gen.Guid(link, False)
pubDate = datetime.fromtimestamp(ckan_module[0].get('x_last_updated_ts', 1000000000), pytz.utc)
item = PyRSS2Gen.RSSItem(title,
link,
description,
None, # author
None, # categories
None, # comments
None, # enclosure
guid,
pubDate,
None) # source
rssitems.append(item)
rss = EnhancedRSS2(
title = INDEX_HTML_HEADER + " feed",
link = LOCAL_URL_PREFIX + "feed/ckan.rss",
description = "The latest ckan recipes",
lastBuildDate = datetime.now(pytz.utc),
items = rssitems)
rss.rss_attrs = {"version": "2.0", "xmlns:atom": "http://www.w3.org/2005/Atom"}
rss_feed_path = os.path.join(FEED_PATH, 'ckan.rss')
with open(rss_feed_path, 'w') as rss_feed_file:
print 'Writing %s' % rss_feed_path
rss.write_xml(rss_feed_file)
print 'Done!'
def main():
print "Building CKAN-API.."
ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH)
latest_versions = {}
all_modules = {}
latest_versions_by_identifier = {}
for ckan_module in ckan_json:
identifier = ckan_module[0]["identifier"]
version = ckan_module[0]["version"]
py_version = LooseVersion(version)
if identifier in latest_versions:
py_version2 = LooseVersion(latest_versions[identifier])
if py_version > py_version2:
print "Version %s > %s" % (py_version, py_version2)
latest_versions[identifier] = version
latest_versions_by_identifier[identifier] = ckan_module
else:
latest_versions[identifier] = version
latest_versions_by_identifier[identifier] = ckan_module
for ckan_module in ckan_json:
identifier = ckan_module[0]["identifier"]
print "Generating API for module %s" % identifier
root_path = os.path.join(API_PATH, identifier)
if not os.path.exists(root_path):
os.makedirs(root_path)
version = ckan_module[0]["version"]
last_modified = "unknown"
try:
last_modified = str(parse(ckan_module[0]["x_last_updated"]))
except:
pass
if identifier in all_modules:
all_modules[identifier] += [{"version": version, "path": "/%s/%s" % (identifier, version)}]
else:
all_modules[identifier] = [{"version": version, "path": "/%s/%s" % (identifier, version)}]
ckan_path = ckan_module[1]
version_path = os.path.join(root_path, version)
if os.path.exists(version_path):
if os.path.isdir(version_path):
os.removedirs(version_path)
else:
os.remove(version_path)
local_path = os.path.join(LOCAL_CKAN_PATH, ckan_path)
print "Symlink: %s -> %s" % (local_path, version_path)
if os.path.exists(version_path):
os.remove(version_path)
os.symlink(local_path, version_path)
for identifier, version in latest_versions.iteritems():
root_path = os.path.join(API_PATH, identifier)
version_path = os.path.join(root_path, version)
latest_path = os.path.join(root_path, "latest")
if identifier in all_modules:
all_modules[identifier] += [{"version": "latest", "path": "/%s/%s" % (identifier, version)}]
else:
all_modules[identifier] = [{"version": "latest", "path": "/%s/%s" % (identifier, version)}]
print "Symlink: %s -> %s" % (version_path, latest_path)
if os.path.exists(latest_path):
os.remove(latest_path)
os.symlink(version_path, latest_path)
all_path = os.path.join(API_PATH, "all")
with open(all_path, "w") as all_file:
print "Writing %s" % all_path
json.dump(all_modules, all_file)
last_modified = {}
for identifier, version in latest_versions.iteritems():
ckan_module = latest_versions_by_identifier[identifier]
parsed_date = None
try:
parsed_date = parse(ckan_module[0]["x_last_updated"])
except:
pass
if parsed_date != None:
now = pytz.utc.localize(datetime.now() - timedelta(hours=24))
if parsed_date > now:
last_modified["/%s/%s" % (identifier, version)] = str(parsed_date)
last_modified = sorted(last_modified.items(), key=operator.itemgetter(1), reverse=True)
pretty_last_modified = []
for url, time in last_modified:
pretty_last_modified += [{"module": url, "last_updated": time}]
latest_path = os.path.join(API_PATH, "latest")
with open(latest_path, "w") as latest_file:
print "Writing %s" % latest_path
json.dump(pretty_last_modified, latest_file)
print "Done!"
