def __call__(self, **kwargs):
merged = {}
merged.update(self.curry)
merged.update(kwargs)
#
# SUPPORT SOME DEFAULT FEATURES CONTROLLED BY TASK PARAMETERS
#
# Allow overriding of prefix via task or cmdline parameter.
# By default use name of the task (no prefix for cleanup tasks)
if "prefix" in merged:
session.setCurrentPrefix(merged["prefix"])
del merged["prefix"]
elif "clean" in self.name:
session.setCurrentPrefix(None)
else:
session.setCurrentPrefix(self.name)
#
# EXECUTE ATTACHED FUNCTION
#
Console.header(self.__name__)
# Execute internal function
return self.func(**merged)
def __call__(self, **kwargs):
merged = {}
merged.update(self.curry)
merged.update(kwargs)
#
# SUPPORT SOME DEFAULT FEATURES CONTROLLED BY TASK PARAMETERS
#
# Allow overriding of prefix via task or cmdline parameter.
# By default use name of the task (no prefix for cleanup tasks)
if "prefix" in merged:
session.setCurrentPrefix(merged["prefix"])
del merged["prefix"]
elif "clean" in self.name:
session.setCurrentPrefix(None)
else:
session.setCurrentPrefix(self.name)
#
# EXECUTE ATTACHED FUNCTION
#
Console.header(self.__name__)
# Execute internal function
return self.func(**merged)
def doCompleteDoctor():
"""Checks for uninstalled or too old versions of requirements and gives a complete output."""
Console.header("Doctor")
dists = [dist for dist in pip.get_installed_distributions()]
keys = [dist.key for dist in pip.get_installed_distributions()]
versions = {}
for dist in dists:
versions[dist.key] = dist.version
def checkSingleInstallation(keys, versions, packageName, minVersion, installPath, updatePath):
Console.info('%s:' % packageName)
Console.indent()
if packageName.lower() in keys:
if LooseVersion(minVersion) > LooseVersion("0.0"):
if LooseVersion(versions[packageName.lower()]) >= LooseVersion(minVersion):
Console.info(Console.colorize('Version is OK (required: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "green"))
else:
Console.info(Console.colorize(Console.colorize('Version installed is too old (required: %s installed: %s)' % (minVersion, versions[packageName.lower()]), "red"), "bold"))
Console.info('Update to the newest version of %s using %s' % (packageName, updatePath))
else:
Console.info(Console.colorize('Found installation', "green"))
else:
Console.info(Console.colorize(Console.colorize('Did NOT find installation', "red"), "bold"))
Console.info('Install the newest version of %s using %s' % (packageName, installPath))
Console.outdent()
# Required packages
Console.info("Required Packages:")
Console.indent()
for entry in needs:
checkSingleInstallation(keys, versions, entry["packageName"], entry["minVersion"], entry["installPath"], entry["updatePath"])
Console.outdent()
# Optional packages
Console.info("")
Console.info("Optional Packages:")
Console.indent()
for entry in optionals:
checkSingleInstallation(keys, versions, entry["packageName"], entry["minVersion"], entry["installPath"], entry["updatePath"])
Console.outdent()
def doCompleteDoctor():
"""Checks for uninstalled or too old versions of requirements and gives a complete output"""
Console.header("Doctor")
dists = [dist for dist in pip.get_installed_distributions()]
keys = [dist.key for dist in pip.get_installed_distributions()]
versions = {}
for dist in dists:
versions[dist.key] = dist.version
def checkSingleInstallation(keys, versions, packageName, minVersion,
installPath, updatePath):
Console.info('%s:' % packageName)
Console.indent()
if packageName.lower() in keys:
Console.info(Console.colorize('Found installation', "green"))
if LooseVersion(minVersion) > LooseVersion("0.0"):
if LooseVersion(versions[packageName.lower()]) >= LooseVersion(
minVersion):
Console.info(
Console.colorize(
'Version is OK (needed: %s installed: %s)' %
(minVersion, versions[packageName.lower()]),
"green"))
else:
Console.info(
Console.colorize(
Console.colorize(
'- Version is NOT OK (needed: %s installed: %s)'
% (minVersion, versions[packageName.lower()]),
"red"), "bold"))
Console.info(
'Update to the newest version of %s using %s' %
(packageName, updatePath))
else:
Console.info(
Console.colorize(
Console.colorize('Did NOT find installation', "red"),
"bold"))
Console.info('Install the newest version of %s using %s' %
(packageName, installPath))
Console.outdent()
# Required packages
Console.info(Console.colorize("Required Packages:", "bold"))
Console.indent()
for entry in needs:
checkSingleInstallation(keys, versions, entry["packageName"],
entry["minVersion"], entry["installPath"],
entry["updatePath"])
Console.outdent()
# Optional packages
Console.info("")
Console.info(Console.colorize("Optional Packages:", "bold"))
Console.indent()
for entry in optionals:
checkSingleInstallation(keys, versions, entry["packageName"],
entry["minVersion"], entry["installPath"],
entry["updatePath"])
Console.outdent()
def setCurrentTask(self, name=None):
if name:
Console.header(name)
self.__currentTask = name
def process(url, start=0, fetch=50):
""" Main processing engine """
pos = start
# End will be updated during each request with incoming data
end = pos + fetch
Console.header("Tumblr Import")
Console.info("Importing data...")
Console.indent()
while pos < end:
Console.info("Requesting %s-%s of %s" % (pos, pos+fetch-1, end))
response = requests.get(url % (pos, fetch))
if response.status_code != 200:
raise Exception("Error during communication with Tumblr: %s" % r.status)
tree = ElementTree.fromstring(response.content)
# This element contains all posts
allPosts = tree.find("posts")
# Update end pointer
end = int(allPosts.get("total"))
# Iterate trough all posts
for post in allPosts:
postType = post.get("type")
postTimeStamp = post.get("unix-timestamp")
postExportDate = str(datetime.datetime.fromtimestamp(int(postTimeStamp)))
postSlug = post.get("slug")
postFormat = post.get("format")
postDateOnly = postExportDate[0:postExportDate.find(" ")]
postFileName = "%s-%s" % (postDateOnly, postSlug)
if postType == "quote":
quoteText = post.find("quote-text").text
quoteComment = post.find("quote-source").text
# Post-process
quoteText = markdownify.markdownify("<blockquote>" + quoteText + "</blockquote>").rstrip("\n").lstrip("\n")
quoteComment = markdownify.markdownify(quoteComment).rstrip("\n")
fileContent = quoteTemplate % (postSlug, postExportDate, quoteText + "\n\n" + quoteComment)
elif postType == "photo":
photoText = post.find("photo-caption").text
try:
photoLinkUrl = post.find("photo-link-url").text
except:
photoLinkUrl = None
photoUrl = post.find("photo-url").text
# Post-process
photoText = markdownify.markdownify(photoText).rstrip("\n")
# Downloading image
photoResponse = requests.get(photoUrl, allow_redirects=True)
if photoResponse.status_code != 200:
Console.error("Unable to load photo. Status: %s; URL: %s", photoResponse.status_code, photoUrl)
continue
# Build extension based on response headers (safer than using file extension)
photoType = photoResponse.headers["content-type"]
if "png" in photoType:
photoExtension = ".png"
elif "jpeg" in photoType or "jpg" in photoType:
photoExtension = ".jpeg"
elif "gif" in photoType:
photoExtension = ".gif"
else:
Console.error("Unknown photo format: %s; Status: %s; URL: %s", photoType, photoResponse.status_code, photoUrl)
continue
# Generating checksum
photoHash = hashlib.sha1(photoResponse.content).hexdigest()
# Generate file name and path from existing data
photoFileName = "%s-%s-%s%s" % (postDateOnly, postSlug, photoHash[0:10], photoExtension)
photoPath = os.path.join(photoFolder, photoFileName)
# Do not repeatly write identical files
if not os.path.exists(photoPath):
photoFile = open(photoPath, "wb")
photoFile.write(photoResponse.content)
photoFile.close()
# Generate basic image tag
photoAsset = '<img src="{{@asset.url %s/%s/%s}}" alt=""/>' % (projectName, photoAssetFolder, photoFileName)
# Wrap with a link when it should be link to an external site
if photoLinkUrl:
photoAsset = '<a href="%s">%s</a>' % (photoLinkUrl, photoAsset)
fileContent = photoTemplate % (postSlug, postExportDate, photoAsset + "\n\n" + photoText)
elif postType == "link":
linkUrl = post.find("link-url").text
try:
linkText = post.find("link-text").text
except:
linkText = linkUrl
# Post-process
if linkText != linkUrl:
linkText = markdownify.markdownify(linkText).rstrip("\n")
fileContent = linkTemplate % (postSlug, postExportDate, "[%s](%s)" % (linkText, linkUrl))
elif postType == "video":
videoCode = post.find("video-source").text
videoText = post.find("video-caption").text
# Post-process
videoText = markdownify.markdownify(videoText).rstrip("\n")
fileContent = videoTemplate % (postSlug, postExportDate, videoCode + "\n\n" + videoText)
elif postType == "regular":
postText = post.find("regular-body").text
try:
postTitle = post.find("regular-title").text
except:
# Ignore posts without title
Console.warn("Ignoring post without title!")
continue
postText = markdownify.markdownify(postText).rstrip("\n")
fileContent = regularTemplate % (postSlug, postExportDate, postTitle, postText)
else:
Console.warn("Unknown POST-TYPE: %s" % postType)
print(ElementTree.dump(post))
continue
# Write post file
fileHandle = open(os.path.join(postFolder, postDateOnly + "-" + postType + "-" + postSlug + ".markdown"), "w")
fileHandle.write(fileContent)
fileHandle.close()
# Update for next requests
pos = pos + fetch
Console.outdent()
Console.info("Successfully imported")
