def main(): ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH) index = '<html><head></head><body>' index += INDEX_HTML_HEADER + '<br/> <br/>' index += '<a href="' + LOCAL_URL_PREFIX + 'master.zip">master.zip</a><br/>' index += '<a href="' + LOCAL_URL_PREFIX + 'ckan.exe">ckan.exe</a> | <a href="' + LOCAL_URL_PREFIX + 'netkan.exe">netkan.exe</a><br/>' index += '<a href="' + LOCAL_URL_PREFIX + 'log.txt">MirrorKAN log</a><br/> <br/>' info = log.getInfo() index += '<font style="color: #669900; font-weight: normal;">Info:<br/>' for item in info: index += item + '<br/>' index += '</font><br/>' warnings = log.getWarnings() index += '<font style="color: #CC3300; font-weight: normal;">Warnings:<br/>' for item in warnings: index += item + '<br/>' index += '</font><br/>' errors = log.getErrors() index += '<font style="color: #CC3300; font-weight: bold;">Errors:<br/>' for item in errors: index += item + '<br/>' index += '</font><br/><br/>' index += 'Modules list:<br/>' for ckan_module in ckan_json: identifier = ckan_module[0]['identifier'] print 'Generating info for module %s' % identifier version = ckan_module[0]['version'] filename = ckan_module[0]['x_mirrorkan_cached_filename'] file_status = ckan_module[0]['x_mirrorkan_download_status'] style = "color: #339900;" if file_status is DLRESULT_HTTP_ERROR_NOT_CACHED: style = "color: #CC3300; font-weight: bold;" elif file_status is DLRESULT_HTTP_ERROR_CACHED: style = "color: #FFD700; font-weight: bold;" index += '<font style="' + style + '">' index += ' ' + identifier + ' - ' + version + ' - ' index += 'Status: ' + ckan_module[0]['x_mirrorkan_status'] + '(' + str(ckan_module[0]['x_mirrorkan_download_status']) + ') - ' index += 'Last update: ' + ckan_module[0]['x_last_updated'] + '<br/>' index += '</font>' index += '</body></html>' print 'Writing index.html' index_file = open(os.path.join(FILE_MIRROR_PATH, 'index.html'), 'w') index_file.write(index) index_file.close()
def main(): print 'Building CKAN RSS Feed..' ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH) unsorted_feed_path = os.path.join(FEED_PATH, 'unsorted.json') with open(unsorted_feed_path, 'w') as unsorted_feed_file: print 'Writing %s' % unsorted_feed_path json.dump(ckan_json, unsorted_feed_file, indent=4, sort_keys=True) sorted_ckan_json = sorted(ckan_json, key=lambda ckan: ckan[0].get('x_last_updated_ts')) sorted_feed_path = os.path.join(FEED_PATH, 'sorted.json') with open(sorted_feed_path, 'w') as sorted_feed_file: print 'Writing %s' % sorted_feed_path json.dump(sorted_ckan_json, sorted_feed_file, indent=4, sort_keys=True) rssitems = [] module_number = 1000000 for ckan_module in sorted_ckan_json: module_number = module_number + 1 # Fallback for link in case nothing can be determinded link = 'http://kerbalstuff.com/' + str(module_number) title = ckan_module[0]['name'] if 'resources' in ckan_module[0]: if 'kerbalstuff' in ckan_module[0]['resources']: link = ckan_module[0]['resources']['kerbalstuff'] # elif 'homepage' in ckan_module[0]['resources']: # link = ckan_module[0]['resources']['homepage'] elif 'repository' in ckan_module[0]['resources']: link = ckan_module[0]['resources']['repository'] # Make links unique link = link + '#' + str(module_number) description = ckan_module[0]['abstract'] guid = PyRSS2Gen.Guid(link, False) pubDate = datetime.fromtimestamp(ckan_module[0].get('x_last_updated_ts', 1000000000), pytz.utc) item = PyRSS2Gen.RSSItem(title, link, description, None, # author None, # categories None, # comments None, # enclosure guid, pubDate, None) # source rssitems.append(item) rss = EnhancedRSS2( title = INDEX_HTML_HEADER + " feed", link = LOCAL_URL_PREFIX + "feed/ckan.rss", description = "The latest ckan recipes", lastBuildDate = datetime.now(pytz.utc), items = rssitems) rss.rss_attrs = {"version": "2.0", "xmlns:atom": "http://www.w3.org/2005/Atom"} rss_feed_path = os.path.join(FEED_PATH, 'ckan.rss') with open(rss_feed_path, 'w') as rss_feed_file: print 'Writing %s' % rss_feed_path rss.write_xml(rss_feed_file) print 'Done!'
def main(): print "Building CKAN-API.." ckan_files, ckan_json = parse_ckan_metadata_directory(LOCAL_CKAN_PATH) latest_versions = {} all_modules = {} latest_versions_by_identifier = {} for ckan_module in ckan_json: identifier = ckan_module[0]["identifier"] version = ckan_module[0]["version"] py_version = LooseVersion(version) if identifier in latest_versions: py_version2 = LooseVersion(latest_versions[identifier]) if py_version > py_version2: print "Version %s > %s" % (py_version, py_version2) latest_versions[identifier] = version latest_versions_by_identifier[identifier] = ckan_module else: latest_versions[identifier] = version latest_versions_by_identifier[identifier] = ckan_module for ckan_module in ckan_json: identifier = ckan_module[0]["identifier"] print "Generating API for module %s" % identifier root_path = os.path.join(API_PATH, identifier) if not os.path.exists(root_path): os.makedirs(root_path) version = ckan_module[0]["version"] last_modified = "unknown" try: last_modified = str(parse(ckan_module[0]["x_last_updated"])) except: pass if identifier in all_modules: all_modules[identifier] += [{"version": version, "path": "/%s/%s" % (identifier, version)}] else: all_modules[identifier] = [{"version": version, "path": "/%s/%s" % (identifier, version)}] ckan_path = ckan_module[1] version_path = os.path.join(root_path, version) if os.path.exists(version_path): if os.path.isdir(version_path): os.removedirs(version_path) else: os.remove(version_path) local_path = os.path.join(LOCAL_CKAN_PATH, ckan_path) print "Symlink: %s -> %s" % (local_path, version_path) if os.path.exists(version_path): os.remove(version_path) os.symlink(local_path, version_path) for identifier, version in latest_versions.iteritems(): root_path = os.path.join(API_PATH, identifier) version_path = os.path.join(root_path, version) latest_path = os.path.join(root_path, "latest") if identifier in all_modules: all_modules[identifier] += [{"version": "latest", "path": "/%s/%s" % (identifier, version)}] else: all_modules[identifier] = [{"version": "latest", "path": "/%s/%s" % (identifier, version)}] print "Symlink: %s -> %s" % (version_path, latest_path) if os.path.exists(latest_path): os.remove(latest_path) os.symlink(version_path, latest_path) all_path = os.path.join(API_PATH, "all") with open(all_path, "w") as all_file: print "Writing %s" % all_path json.dump(all_modules, all_file) last_modified = {} for identifier, version in latest_versions.iteritems(): ckan_module = latest_versions_by_identifier[identifier] parsed_date = None try: parsed_date = parse(ckan_module[0]["x_last_updated"]) except: pass if parsed_date != None: now = pytz.utc.localize(datetime.now() - timedelta(hours=24)) if parsed_date > now: last_modified["/%s/%s" % (identifier, version)] = str(parsed_date) last_modified = sorted(last_modified.items(), key=operator.itemgetter(1), reverse=True) pretty_last_modified = [] for url, time in last_modified: pretty_last_modified += [{"module": url, "last_updated": time}] latest_path = os.path.join(API_PATH, "latest") with open(latest_path, "w") as latest_file: print "Writing %s" % latest_path json.dump(pretty_last_modified, latest_file) print "Done!"