def importTmpProducts(): util.logNow("RUN : importTmpProducts()") query = """ LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id, category_1,category_2,category_3, name, price, discount, discount_percent, brand, url, url_source, delivery_period, is_available, image_1, image_2, image_3, image_4, image_5, image_zoom, created_at) """.format(config.FILE_CSV_PATH, config.TABLE_TMP_PRODUCT) mysql.executeNoneQuery(query)
def backupDeletedProducts(): util.logNow("RUN : backupDeletedProducts()") query = """ INSERT INTO {0} SELECT * FROM {1} WHERE is_deleted = 1 """.format(config.TABLE_DELETED_PRODUCT, config.TABLE_PRODUCT) return mysql.executeNoneQuery(query)
def main(): print "\n========== RUN get_stat.py ============" util.logNow("START AT") rcb_list = common.getRcbSites() for item in rcb_list: print("=========> PAGE: ", item[1]) if item[2] == "invest-tracing.com": invest_tracing.run(item) elif item[2] == "hyip-cruiser.com": hyip_cruiser.run(item) elif item[2] == "hyipscope.org": hyipscope.run(item) elif item[2] == "graspgold.com": graspgold.run(item) elif item[2] == "myhyips.net": myhyips.run(item) elif item[2] == "hyipdollar.com": hyipdollar.run(item) elif item[2] == "makeindmoney.com": makeindmoney.run(item) elif item[2] == "myinvestblog.ru": myinvestblog.run(item) elif item[2] == "uhyips.com": uhyips.run(item) common.statsRcbDaily() util.logNow("END AT")
def main(): print "\n========== RUN get_data_feed.py ============" util.logNow("START AT") downloadXMLFile() parseXMLToCSV() util.logNow("END AT")
def insertNewProducts(): util.logNow("RUN : insertNewProducts()") query = """ INSERT IGNORE INTO {0} SELECT * FROM {1} """.format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT) mysql.executeNoneQuery(query)
def setIsDeleted(): util.logNow("RUN : setIsDeleted()") query = """ UPDATE {0} SET is_deleted = 1, deleted_at = now() WHERE is_deleted = 0 AND id NOT IN (SELECT id FROM {1}) """.format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT) return mysql.executeNoneQuery(query)
def truncateTmpProducts(): util.logNow("RUN : truncateProduct()") query = "TRUNCATE TABLE {0}".format(config.TABLE_TMP_PRODUCT) try: print query cursor.execute(query) db.commit() except Exception as e: print(e) db.rollback()
def downloadXMLFile(): util.logNow("Run downloadXMLFile()") response = urllib2.urlopen(config.DATA_FEED_URL) CHUNK = 16 * 1024 with open(config.FILE_XML_PATH, 'wb') as f: while True: chunk = response.read(CHUNK) if not chunk: break f.write(chunk)
def parseXMLToCSV(): global writer util.logNow("Run parseXMLToCSV()") writer = open(config.FILE_CSV_PATH, 'w') context = etree.iterparse( config.FILE_XML_PATH, tag='Product' ) fast_iter(context, process_element) writer.close()
def main(): print "\n========== RUN check_rcb.py ============" util.logNow("START AT") siteMonitors = getSiteMonitor() for item in siteMonitors: print(item) if item[2] == 0: if checkPaid(item[1]): setPaid(item[0]) util.logNow("END AT")
def main(): print "\n========== RUN get_stat.py ============" util.logNow("START AT") rcb_list = common.getRcbSites() for item in rcb_list: if item[2] == "uhyips.com": uhyips.run(item) common.statsRcbDaily() util.logNow("END AT")
def main(): print "\n========== RUN get_stat.py ============" util.logNow("START AT") unitedswissbanks_com.run() depdun_com.run() oktrade_biz.run() lucrativeventure_co.run() becomerich.run() oilenergy.run() util.logNow("END AT")
def main(): print "\n=========== RUN delete_products.py ============" util.logNow("START AT") setIsDeleted() count = countDeletedProducts() print "COUNT : ", count if count < config.LIMIT_DELETED: if backupDeletedProducts(): removeDeletedProducts() util.logNow("END AT")
def importTmpProducts(): util.logNow("RUN : importTmpProducts()") query = """ LOAD DATA INFILE '{0}' INTO TABLE {1} FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (id, category_1,category_2,category_3, name, price, discount, discount_percent, brand, url, url_source, delivery_period, is_available, image_1, image_2, image_3, image_4, image_5, image_zoom, created_at) """.format(config.FILE_CSV_PATH, config.TABLE_TMP_PRODUCT) try: print query cursor.execute(query) db.commit() except Exception as e: print(e) db.rollback()
def main(): print "\n=========== RUN import_to_db.py ============" util.logNow("START AT") truncateTmpProducts() importTmpProducts() updateProducts() insertNewProducts() # updateIsAvailable() util.logNow("END AT")
def updateProducts(): util.logNow("RUN : updateProducts()") query = """ UPDATE {0} p JOIN {1} tp ON tp.id = p.id SET p.price = tp.price, p.discount = tp.discount, p.discount_percent = tp.discount_percent """.format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT) mysql.executeNoneQuery(query)
def insertNewProducts(): util.logNow("RUN : insertNewProducts()") query = """ INSERT IGNORE INTO {0} SELECT * FROM {1} """.format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT) try: print query cursor.execute(query) db.commit() except Exception as e: print(e) db.rollback()
def updateProducts(): util.logNow("RUN : updateProducts()") query = """ UPDATE {0} p JOIN {1} tp ON tp.id = p.id SET p.price = tp.price, p.discount = tp.discount, p.discount_percent = tp.discount_percent """.format(config.TABLE_PRODUCT, config.TABLE_TMP_PRODUCT) try: print query cursor.execute(query) db.commit() except Exception as e: print(e) db.rollback()
def main(): print "\n========== RUN get_rcb.py ============" util.logNow("START AT") hyip_scope.run() my_hyip.run() invest_tracing.run() my_invest_blog.run() list4hyip_com.run() hyip_cruiser_com.run() graspgold.run() makeindmoney.run() hyipdollar_com.run() gs_monitor.run() hyip_income.run() uhyips.run() hyip_tank.run() # hyip_stop.run() bloghyip_com.run() # goodhyip_biz.run() #monhyip.run() util.logNow("END AT")
def truncateTmpProducts(): util.logNow("RUN : truncateTmpProducts()") query = "TRUNCATE TABLE {0}".format(config.TABLE_TMP_PRODUCT) mysql.executeNoneQuery(query)
def removeDeletedProducts(): util.logNow("RUN : removeDeletedProducts()") query = """ DELETE FROM {0} WHERE is_deleted = 1 """.format(config.TABLE_PRODUCT) mysql.executeNoneQuery(query)
def main(): print "\n========== RUN test_get_stat.py ============" util.logNow("START AT") oilenergy.run() util.logNow("END AT")
def countDeletedProducts(): util.logNow("RUN : countDeletedProducts()") query = """ SELECT COUNT(*) FROM {0} WHERE is_deleted = 1 """.format(config.TABLE_PRODUCT) return mysql.executeScalar(query)
def main(): print "\n========== RUN test_get_rcb.py ============" util.logNow("START AT") uhyips.run() util.logNow("END AT")
query = """ SELECT id, url_source, is_available FROM {0} WHERE status = 'Approved' """.format(config.TABLE_PRODUCT) try: results = mysql.executeQuery(query) for row in results: status = util.checkURL(row[1]) if status != row[2]: setAvailable(row[0], status) except Exception as e: print(e) def main(): updateIsAvailable() ###### MAIN ###### print "\n=========== RUN import_to_db.py ============" mysql.connect() util.logNow("START AT") main() util.logNow("END AT") mysql.disconnect()