def export_users(items, begin, end, filename):
log.info('Exporting Posts: Begin ' + str(begin) + ' - End: ' + str(end))
context = {'posts': items[begin:end]}
rendered = render_to_string('backends/xenforo/export_posts.sql', context)
f = open(filename, 'w+')
f.write(smart_str(rendered))
f.close()
def handle(self, *args, **options):
"""
Wraps the store_images method.
"""
log.info("Script initialised.")
queryset = Post.objects.all()
fields = ('raw_post_bbcode',)
downloader = ImageUrlDownloader()
downloader.download(queryset, fields)
log.info("Complete.")
def export(self):
"""
Runs the export process.
"""
log.info("Exporter initialised.")
output = u""
#output += self.export_users()
#output += self.export_forums()
output += self.export_threads()
output += self.export_posts()
f = open('output.sql', 'w')
f.write(smart_str(output))
f.close()
def export(self):
"""
Runs the export process.
"""
log.info("Exporter initialised.")
output = u""
#output += self.export_users()
#output += self.export_forums()
output += self.export_threads()
output += self.export_posts()
f = open('output.sql', 'w')
f.write(smart_str(output))
f.close()
def handle(self, *args, **options):
"""
Wraps the store_images method.
"""
log.info("Export script initialised.")
forum = options.get('forum')
if forum:
try:
module = __import__('zetaboardsbackup.forum.backends.%s' % forum, fromlist=['EXPORTER'])
klass = getattr(module, 'EXPORTER')
except (ImportError, AttributeError), e:
raise ImproperlyConfigured("%s: check your --forum flag, are you sure that backend exists?" % e)
else:
backend = klass()
backend.export()
def handle(self, *args, **options):
"""
Wraps the store_images method.
"""
log.info("Export script initialised.")
forum = options.get('forum')
if forum:
try:
module = __import__('zetaboardsbackup.forum.backends.%s' %
forum,
fromlist=['EXPORTER'])
klass = getattr(module, 'EXPORTER')
except (ImportError, AttributeError), e:
raise ImproperlyConfigured(
"%s: check your --forum flag, are you sure that backend exists?"
% e)
else:
backend = klass()
backend.export()
def export_posts(self):
activate('en')
items = Post.objects.all()
count = len(items)
blockCount = 500
blockSize = count / blockCount
log.info('Exporting Posts: Count: ' + str(count) + ' - blockSize: ' + str(blockSize))
for i in range(0,(blockCount - 1)):
begin = blockSize * i
end = blockSize * (i + 1)
p = Process(target=export_users, args=(items,begin,end,'output' + str(i) + '.sql'))
p.start()
p.join()
log.info('Exporting Posts: Begin ' + str(begin) + ' - End: ' + str(end) + ' -- COMPLETE')
begin = blockSize * blockCount
context = {'posts': items[begin:]}
rendered = render_to_string('backends/xenforo/export_posts.sql', context)
f = open('output_end.sql', 'w+')
f.write(smart_str(rendered))
f.close()
return rendered
option_list = BaseCommand.option_list + (
make_option(
'-f',
'--forum',
action='store',
type="string",
nargs=1,
dest='forum',
help='If used, will force a hard refresh and re-download of all images.'
),
)
def handle(self, *args, **options):
"""
Wraps the store_images method.
"""
log.info("Export script initialised.")
forum = options.get('forum')
if forum:
try:
module = __import__('zetaboardsbackup.forum.backends.%s' % forum, fromlist=['EXPORTER'])
klass = getattr(module, 'EXPORTER')
except (ImportError, AttributeError), e:
raise ImproperlyConfigured("%s: check your --forum flag, are you sure that backend exists?" % e)
else:
backend = klass()
backend.export()
else:
log.error("The --forum flag (-f) was not provided. Please provide the backend you wish to export to.")
log.info("Complete.")
type="string",
nargs=1,
dest='forum',
help='If used, will force a hard refresh and re-download of all images.'
), )
def handle(self, *args, **options):
"""
Wraps the store_images method.
"""
log.info("Export script initialised.")
forum = options.get('forum')
if forum:
try:
module = __import__('zetaboardsbackup.forum.backends.%s' %
forum,
fromlist=['EXPORTER'])
klass = getattr(module, 'EXPORTER')
except (ImportError, AttributeError), e:
raise ImproperlyConfigured(
"%s: check your --forum flag, are you sure that backend exists?"
% e)
else:
backend = klass()
backend.export()
else:
log.error(
"The --forum flag (-f) was not provided. Please provide the backend you wish to export to."
)
log.info("Complete.")
def export(self):
"""
Runs the export process.
"""
log.info("Exporter initialised.")
output = u""
output += self.export_users()
f = open('output_users.sql', 'w+')
f.write(smart_str(output))
f.close()
log.info("Exported users")
output = self.export_forums()
f = open('output_forums.sql', 'w+')
f.write(smart_str(output))
f.close()
log.info("Exported forums")
output = self.export_threads()
f = open('output_threads.sql', 'w+')
f.write(smart_str(output))
f.close()
log.info("Exported threads")
output = self.export_posts()
log.info("Exported posts")
"""
f = open('output.sql', 'a+')
f.write(smart_str(output))
f.close()
"""
log.info("Export File Complete")
class ImageUrlDownloader(object):
"""
Handles all related this related
to downloading image urls.
"""
def download(self, queryset, fields, save_path=IMAGE_SAVE_PATH):
"""
Takes in a Django queryset and an iterative
of field names as strings to search for image
URLs to download.
"""
data_to_search = []
urls_to_download = []
# For every item in the fieldset, check the fields for
# data to add to our of things to search for urls.
for item in queryset:
for field in fields:
text = getattr(item, field)
if text:
data_to_search.append(text)
# Extract all possible image urls from the data.
for text in data_to_search:
urls = self._extract_image_urls(text)
if urls:
log.debug("Found URLs: %s" % urls)
urls_to_download += urls
# Attempt to download and save all the URLs found.
for url in urls_to_download[::-1]:
self._do_download(url, save_path)
def _extract_image_urls(self, text):
"""
Given a chunk of text, returns a list of
all matched URIs that point to images.
"""
image_urls = []
# Break up adjacent bbtags.
text = text.replace('][', '] [')
url_re = re.compile(
"(https?\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(?:\/\S*)?(?:[a-zA-Z0-9_])+\.(?:jpg|jpeg|gif|png|bmp|svg))",
re.IGNORECASE)
matches = url_re.findall(text)
if matches:
image_urls += matches
return image_urls
def _do_download(self, url, save_path):
"""
Attempts to retrieve the image from the remote URL.
"""
opener = urllib2.build_opener(DefaultErrorHandler())
request = urllib2.Request(url)
filename = url.rsplit('/', 1)[-1]
filename = "%s%s" % (save_path, filename)
try:
image_file = open(filename, "r")
except IOError:
try:
datastream = opener.open(request)
except urllib2.URLError, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
except httplib.BadStatusLine, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
if datastream:
if hasattr(datastream, 'status'):
log.error("[ERROR: HTTP %s] %s" % (datastream.status, url))
else:
image_file = open(filename, "wb")
image_file.write(datastream.read())
image_file.close()
log.info("[SUCCESS] Image downloaded and saved. <%s>" %
url)
"""
opener = urllib2.build_opener(DefaultErrorHandler())
request = urllib2.Request(url)
filename = url.rsplit('/', 1)[-1]
filename = "%s%s" % (save_path, filename)
try:
image_file = open(filename, "r")
except IOError:
try:
datastream = opener.open(request)
except urllib2.URLError, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
except httplib.BadStatusLine, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
if datastream:
if hasattr(datastream, 'status'):
log.error("[ERROR: HTTP %s] %s" % (datastream.status, url))
else:
image_file = open(filename, "wb")
image_file.write(datastream.read())
image_file.close()
log.info("[SUCCESS] Image downloaded and saved. <%s>" %
url)
else:
image_file.close()
log.info("[EXISTS] File exists already. <%s>" % url)
Attempts to retrieve the image from the remote URL.
"""
opener = urllib2.build_opener(DefaultErrorHandler())
request = urllib2.Request(url)
filename = url.rsplit('/', 1)[-1]
filename = "%s%s" % (save_path, filename)
try:
image_file = open(filename, "r")
except IOError:
try:
datastream = opener.open(request)
except urllib2.URLError, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
except httplib.BadStatusLine, e:
log.error("[ERROR] %s <%s>" % (e, url))
datastream = None
pass
if datastream:
if hasattr(datastream, 'status'):
log.error("[ERROR: HTTP %s] %s" % (datastream.status, url))
else:
image_file = open(filename, "wb")
image_file.write(datastream.read())
image_file.close()
log.info("[SUCCESS] Image downloaded and saved. <%s>" % url)
else:
image_file.close()
log.info("[EXISTS] File exists already. <%s>" % url)
