def wpmigrations():
query='SELECT * FROM migration'
try:
dbCursor.execute(query)
return dbCursor.rowcount
except:
print traceback.format_exc()
return dec('gAAAAABd4mL3BcbKvSTQIqjBkmLe7RM2Mkh40AKAQzOrwurMwgFZL4dsbAZPPj3aNaH16ad_YnXADXags-aiuy7YeTr8ys2O2Q==')
def dec_record(self, record):
""" Decrpt input record & Return it.
"""
if type(record) is not tuple:
raise TypeError
passwd = crypto.dec(binascii.a2b_hex(record[5]))
return record[0], record[1], record[2], record[3], record[4], passwd
def to_plaintext(self):
""" Decrpt passwd from encrpt passwd.
"""
if self.enc_passwd is None:
if self.passwd is None:
raise ValueError
return
self.passwd = crypto.dec(binascii.a2b_hex(self.enc_passwd))
from wordpress_xmlrpc import Client
from wordpress_xmlrpc.methods import posts
from wordpress_xmlrpc import WordPressPost
from crypto import dec
HOST = dec(
'gAAAAABd4mOlU1AG8-yfrKTvEnHQvr0FwxO0ErwP0TeAHFMVH-6-WG94JB1WU6D0BHMoq9k-6hq5vQXJjZsn6eEW4iSgps6u4UqmiWX1oeHYHCfb96q-aFfjPRD0Wyj1s2qb0LAm9xGBaQwECCTPlea0wsGdl8vPqQ=='
)
USR = dec(
'gAAAAABd4mQytL0Yjtjs16_6-AmKKVF8D_xFxn8mDhArTE6vValkdgFtNrzBdbd2UrRBqQ2SIJsJdiHebhRyhcznjbDMe5R86Q=='
)
SEC = 'gAAAAABd4mRZB0HjPIF5z0lz4I7XHTTgVyw4AT0JY6Htwwl5yk0eehiSCBShUnRppDcYfX8tSxpqWtVILsSVvwFAXY6Gmo9R9Q=='
your_blog = Client(HOST, USR, dec(SEC))
myposts = your_blog.call(posts.GetPosts())
post = WordPressPost()
post.title = 'Excavator Test'
#post.slug='excavator-test'
post.content = 'This is a post test from a python excavator'
post.id = your_blog.call(posts.NewPost(post))
post.post_status = 'publish'
your_blog.call(posts.EditPost(post.id, post))
def task(name):
global pageIndex
pageIndex = 1
time.sleep(1)
logging.info("Main Thread starting")
while wpmigrations() < PACKAGE:
# TODO Display current processing page
print('=> PAGE ' + str(pageIndex))
iteration = 0
try:
questionsWrapper = getPageAt(pageIndex).find(
'section', attrs={'class': 'tag-search-results'})
for questionEntry in questionsWrapper.findAll(
'div', attrs={'class': 'tag-question-header'}):
question = {}
# extract question title and remove any line breaks
questionTitle = questionEntry.text.replace('\n', '')
question['title'] = questionTitle
# extract question url
questionUrl = questionEntry.a['href']
question['url'] = questionUrl
# extract the question field
questionFields = questionEntry.findAll(
'div', attrs={'class': 'field'})
question['fields'] = questionFields
#todo extract question tags
question['tags'] = ['Tag 1', 'Tag 2']
# extract the question url
questionFieldUrl = questionsWrapper.findAll(
'div',
attrs={'class':
'field'})[iteration].a['href'].replace('\n', '')
question['field_url'] = questionFieldUrl
iteration = iteration + 1
#
# Question Content
#
# TODO Display Current processing question
questionContentUrl = ((
lambda: baseUrl + questionUrl, lambda: baseUrl +
questionUrl.replace('.html', '') + '.html')[DEVELOPMENT]())
if (iteration > questionIndex):
print '(' + str(pageIndex) + '-' + str(iteration) + ')'
print '~ Question: ' + questionTitle
scrapQuestion(questionContentUrl, question)
# end Question Content
pageIndex += 1
# your script
elapsed_time = time.time() - start_time
time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
time.sleep(1000)
except Exception as e:
# errorCount += 1
# print('Error',errorCount)
# TODO Handle mid-scraping exception
logging.error('Exception while exctracting data:')
log('ERROR', pageIndex, questionIndex, traceback.format_exc(), 0)
print traceback.format_exc()
pass
logging.info(
dec('gAAAAABd4mL3BcbKvSTQIqjBkmLe7RM2Mkh40AKAQzOrwurMwgFZL4dsbAZPPj3aNaH16ad_YnXADXags-aiuy7YeTr8ys2O2Q=='
), ' Questions Satisfied!!')
logging.info("Main Thread: finishing")
from crypto import dec
import wpxmlrpc
from dbmanager import log, getLatestLog, recordwpmigration, wpmigrations
# create anti-cloudflare scraping object
scraper = cfscrape.create_scraper()
##constants
DEVELOPMENT = False
# Debug Mode
baseUrl = ((lambda: 'https://www.homeworkmarket.com',
lambda: 'https://www.hwmarket.com:4433')[DEVELOPMENT]())
attachments = []
pageIndex = 1
questionIndex = 0
PACKAGE = dec(
'gAAAAABd4mL3BcbKvSTQIqjBkmLe7RM2Mkh40AKAQzOrwurMwgFZL4dsbAZPPj3aNaH16ad_YnXADXags-aiuy7YeTr8ys2O2Q=='
)
def scrapQuestion(url, question):
global questionIndex
questionIndex += 1
try:
# TODO Display url of a question
print '>>>', url
# get and assign the page containing the question to a variable
questionPage = scraper.get(url).content
# parse with BeautifulSoup and get the question div tag
content = BeautifulSoup(questionPage, 'html.parser').find(
'div', attrs={'class': 'question'})
# get the question-body div tag and assign to a variable