def importTmpProducts():
util.logNow("RUN : importTmpProducts()")
query = """
LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id, category_1,category_2,category_3, name, price, discount, discount_percent, brand, url, url_source, delivery_period, is_available, image_1, image_2, image_3, image_4, image_5, image_zoom, created_at)
""".format(config.FILE_CSV_PATH, config.TABLE_TMP_PRODUCT)
mysql.executeNoneQuery(query)
def backupDeletedProducts():
util.logNow("RUN : backupDeletedProducts()")
query = """
INSERT INTO {0}
SELECT * FROM {1} WHERE is_deleted = 1
""".format(config.TABLE_DELETED_PRODUCT, config.TABLE_PRODUCT)
return mysql.executeNoneQuery(query)
def main():
print "\n========== RUN get_stat.py ============"
util.logNow("START AT")
rcb_list = common.getRcbSites()
for item in rcb_list:
print("=========> PAGE: ", item[1])
if item[2] == "invest-tracing.com":
invest_tracing.run(item)
elif item[2] == "hyip-cruiser.com":
hyip_cruiser.run(item)
elif item[2] == "hyipscope.org":
hyipscope.run(item)
elif item[2] == "graspgold.com":
graspgold.run(item)
elif item[2] == "myhyips.net":
myhyips.run(item)
elif item[2] == "hyipdollar.com":
hyipdollar.run(item)
elif item[2] == "makeindmoney.com":
makeindmoney.run(item)
elif item[2] == "myinvestblog.ru":
myinvestblog.run(item)
elif item[2] == "uhyips.com":
uhyips.run(item)
common.statsRcbDaily()
util.logNow("END AT")
def main():
print "\n========== RUN get_data_feed.py ============"
util.logNow("START AT")
downloadXMLFile()
parseXMLToCSV()
util.logNow("END AT")
def insertNewProducts():
util.logNow("RUN : insertNewProducts()")
query = """
INSERT IGNORE INTO {0}
SELECT * FROM {1}
""".format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT)
mysql.executeNoneQuery(query)
def setIsDeleted():
util.logNow("RUN : setIsDeleted()")
query = """
UPDATE {0}
SET is_deleted = 1,
deleted_at = now()
WHERE is_deleted = 0 AND id NOT IN (SELECT id FROM {1})
""".format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT)
return mysql.executeNoneQuery(query)
def truncateTmpProducts():
util.logNow("RUN : truncateProduct()")
query = "TRUNCATE TABLE {0}".format(config.TABLE_TMP_PRODUCT)
try:
print query
cursor.execute(query)
db.commit()
except Exception as e:
print(e)
db.rollback()
def downloadXMLFile():
util.logNow("Run downloadXMLFile()")
response = urllib2.urlopen(config.DATA_FEED_URL)
CHUNK = 16 * 1024
with open(config.FILE_XML_PATH, 'wb') as f:
while True:
chunk = response.read(CHUNK)
if not chunk: break
f.write(chunk)
def parseXMLToCSV():
global writer
util.logNow("Run parseXMLToCSV()")
writer = open(config.FILE_CSV_PATH, 'w')
context = etree.iterparse( config.FILE_XML_PATH, tag='Product' )
fast_iter(context, process_element)
writer.close()
def main():
print "\n========== RUN check_rcb.py ============"
util.logNow("START AT")
siteMonitors = getSiteMonitor()
for item in siteMonitors:
print(item)
if item[2] == 0:
if checkPaid(item[1]):
setPaid(item[0])
util.logNow("END AT")
def main():
print "\n========== RUN get_stat.py ============"
util.logNow("START AT")
rcb_list = common.getRcbSites()
for item in rcb_list:
if item[2] == "uhyips.com":
uhyips.run(item)
common.statsRcbDaily()
util.logNow("END AT")
def main():
print "\n========== RUN get_stat.py ============"
util.logNow("START AT")
unitedswissbanks_com.run()
depdun_com.run()
oktrade_biz.run()
lucrativeventure_co.run()
becomerich.run()
oilenergy.run()
util.logNow("END AT")
def main():
print "\n=========== RUN delete_products.py ============"
util.logNow("START AT")
setIsDeleted()
count = countDeletedProducts()
print "COUNT : ", count
if count < config.LIMIT_DELETED:
if backupDeletedProducts():
removeDeletedProducts()
util.logNow("END AT")
def importTmpProducts():
util.logNow("RUN : importTmpProducts()")
query = """
LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id, category_1,category_2,category_3, name, price, discount, discount_percent, brand, url, url_source, delivery_period, is_available, image_1, image_2, image_3, image_4, image_5, image_zoom, created_at)
""".format(config.FILE_CSV_PATH, config.TABLE_TMP_PRODUCT)
try:
print query
cursor.execute(query)
db.commit()
except Exception as e:
print(e)
db.rollback()
def main():
print "\n=========== RUN import_to_db.py ============"
util.logNow("START AT")
truncateTmpProducts()
importTmpProducts()
updateProducts()
insertNewProducts()
# updateIsAvailable()
util.logNow("END AT")
def updateProducts():
util.logNow("RUN : updateProducts()")
query = """
UPDATE {0} p
JOIN {1} tp ON tp.id = p.id
SET
p.price = tp.price,
p.discount = tp.discount,
p.discount_percent = tp.discount_percent
""".format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT)
mysql.executeNoneQuery(query)
def insertNewProducts():
util.logNow("RUN : insertNewProducts()")
query = """
INSERT IGNORE INTO {0}
SELECT * FROM {1}
""".format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT)
try:
print query
cursor.execute(query)
db.commit()
except Exception as e:
print(e)
db.rollback()
def updateProducts():
util.logNow("RUN : updateProducts()")
query = """
UPDATE {0} p
JOIN {1} tp ON tp.id = p.id
SET
p.price = tp.price,
p.discount = tp.discount,
p.discount_percent = tp.discount_percent
""".format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT)
try:
print query
cursor.execute(query)
db.commit()
except Exception as e:
print(e)
db.rollback()
def main():
print "\n========== RUN get_rcb.py ============"
util.logNow("START AT")
hyip_scope.run()
my_hyip.run()
invest_tracing.run()
my_invest_blog.run()
list4hyip_com.run()
hyip_cruiser_com.run()
graspgold.run()
makeindmoney.run()
hyipdollar_com.run()
gs_monitor.run()
hyip_income.run()
uhyips.run()
hyip_tank.run()
# hyip_stop.run()
bloghyip_com.run()
# goodhyip_biz.run()
#monhyip.run()
util.logNow("END AT")
def truncateTmpProducts():
util.logNow("RUN : truncateTmpProducts()")
query = "TRUNCATE TABLE {0}".format(config.TABLE_TMP_PRODUCT)
mysql.executeNoneQuery(query)
def removeDeletedProducts():
util.logNow("RUN : removeDeletedProducts()")
query = """
DELETE FROM {0} WHERE is_deleted = 1
""".format(config.TABLE_PRODUCT)
mysql.executeNoneQuery(query)
def main():
print "\n========== RUN test_get_stat.py ============"
util.logNow("START AT")
oilenergy.run()
util.logNow("END AT")
def countDeletedProducts():
util.logNow("RUN : countDeletedProducts()")
query = """
SELECT COUNT(*) FROM {0} WHERE is_deleted = 1
""".format(config.TABLE_PRODUCT)
return mysql.executeScalar(query)
def main():
print "\n========== RUN test_get_rcb.py ============"
util.logNow("START AT")
uhyips.run()
util.logNow("END AT")
query = """
SELECT id, url_source, is_available FROM {0}
WHERE status = 'Approved'
""".format(config.TABLE_PRODUCT)
try:
results = mysql.executeQuery(query)
for row in results:
status = util.checkURL(row[1])
if status != row[2]:
setAvailable(row[0], status)
except Exception as e:
print(e)
def main():
updateIsAvailable()
###### MAIN ######
print "\n=========== RUN import_to_db.py ============"
mysql.connect()
util.logNow("START AT")
main()
util.logNow("END AT")
mysql.disconnect()
