def experiment_MainObj_main_download_listings():
"""Experiment with DaemonMain.run_daemon"""
from clair.daemon import DaemonMain
print "start"
conf_dir = relative("../../../../clair-data")
data_dir = relative("../../../../clair-data")
# conf_dir = relative("../../example-data-1")
# data_dir = relative("../../example-data-1")
m = DaemonMain(conf_dir, data_dir)
m.run_daemon(-1)
print "finished!"
def test_MainObj_main_download_listings():
"""Test DaemonMain.run_daemon"""
from clair.daemon import DaemonMain
example_data = relative("../../example-data")
conf_dir = relative("../../test-data/test-download-listings")
data_dir = relative("../../test-data/test-download-listings")
#Create a fresh copy of the example data
os.system("rm -rf " + data_dir)
os.system("cp -r " + example_data + " " + data_dir)
m = DaemonMain(conf_dir, data_dir)
m.run_daemon(1)
print "finished!"
def test_MainObj_compute_next_due_time():
"""Test DaemonMain.compute_next_due_time"""
from clair.daemon import DaemonMain
m = DaemonMain("..", "..")
#Without random the new times must be on the start of the interval
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 0, 0), "monthly")
print t1
assert t1 == datetime(2000, 2, 1, 0, 0)
t1 = m.compute_next_due_time(datetime(2000, 1, 10, 20, 15), "monthly")
print t1
assert t1 == datetime(2000, 2, 1, 0, 0)
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 20, 15), "weekly")
print t1
assert t1 == datetime(2000, 1, 3, 0, 0) #Monday 2000-1-3
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 0, 0), "daily")
print t1
assert t1 == datetime(2000, 1, 2, 0, 0)
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 20, 15), "daily")
print t1
assert t1 == datetime(2000, 1, 2, 0, 0)
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 20, 0), "hourly")
print t1
assert t1 == datetime(2000, 1, 1, 21, 0)
t1 = m.compute_next_due_time(datetime(2000, 1, 1, 20, 15), "hourly")
print t1
assert t1 == datetime(2000, 1, 1, 21, 0)
# t = datetime(2000, 1, 1, 20, 15)
# times = [t]
# for _ in range(20):
# t = m.compute_next_due_time(t, "day", True)
# times.append(t)
## print times
# times = pd.Series(times)
# print times
# diff = times.diff()
# print diff
print "finished!"
def test_MainObj_execute_tasks():
"""Test DaemonMain.execute_tasks"""
from clair.daemon import DaemonMain
from clair.coredata import Product, SearchTask, UpdateTask
conf_dir = relative("..")
data_dir = None
#Create product information and search tasks
m = DaemonMain(conf_dir, data_dir)
m.data.set_products([Product("nikon-d90", "Nikon D90", "DSLR Camera",
None, None),
Product("nikon-d70", "Nikon D70", "DSLR Camera",
None, None)])
m.data.add_tasks([SearchTask("s-nikon-d90", datetime(2000,1,1), "ebay-de",
"Nikon D90", "daily", 5, 150, 500, "EUR",
["nikon-d90"]),
SearchTask("s-nikon-d90-2", datetime(2000,1,1), "ebay-de",
"Nikon D90", "daily", 5, 150, 500, "EUR",
["nikon-d90"]),
SearchTask("s-nikon-d70", datetime(2000,1,1), "ebay-de",
"Nikon D70", "daily", 5, 50, 500, "EUR",
["nikon-d70"])])
#Execute the search tasks
m.execute_tasks()
print m.data.tasks
print m.data.listings[["title", "price", "sold", "time"]].to_string()
#Create an update task, that updates all listings immediately,
# and is executed only once
upd_listings = m.data.listings["id"]
m.data.add_tasks([UpdateTask("update-1", datetime(2000,1,1), "ebay-de",
None, upd_listings)])
#Execute the update task
m.execute_tasks()
print m.data.tasks
print m.data.listings[["title", "price", "sold", "time", "server"]].to_string()
# print m.listings[["description", "price"]].to_string()
wakeup_time, sleep_sec = m.compute_next_wakeup_time()
print "wakeup_time:", wakeup_time, "sleep_sec:", sleep_sec
#The search tasks must still exist, but the update task must be deleted
assert len(m.data.tasks) == 3
#There must be about 10 listings, 5 from each search task,
#but "s-nikon-d90" and "s-nikon-d90-2" have the same search string, and
#should find the same listings
assert 8 <= len(m.data.listings) <= 12 #fewer listings: variants of the same item are removed
#Test correct behavior if multiple search tasks find the same listings:
#There is a 2nd ``SearchTask`` that also searches for "Nikon D90",
#but with different name: "s-nikon-d90-2".
#This this task must appear in the "search_tasks" field.
tasks_lens = [0, 0, 0, 0] #Histogram of length of "search_tasks" list
prods_lens = [0, 0, 0, 0] #Histogram of length of "expected_products" list
n_2_d90 = 0 #Number of listings that were found by "s-nikon-d90" and "s-nikon-d90-2".
for _, listing in m.data.listings.iterrows():
# print listing
#Compute histogram of length of "search_tasks" list
n_tasks = len(listing["search_tasks"])
tasks_lens[n_tasks] += 1
#Compute histogram of length of "search_tasks" list
n_prods = len(listing["expected_products"])
prods_lens[n_prods] += 1
#how many listings have "s-nikon-d90" and "s-nikon-d90-2" in "search_tasks" field
if "s-nikon-d90" in listing["search_tasks"] and \
"s-nikon-d90-2" in listing["search_tasks"]:
n_2_d90 += 1
#These tests may sometimes fail, some listings contain D90 and D70 in
#their title.
print "tasks_lens", tasks_lens
assert tasks_lens[0] == 0 #listings without a search task: impossible
assert 4 <= tasks_lens[1] <= 5 #listings with one search task: D70
assert 4 <= tasks_lens[2] <= 5 #listings with two search tasks:
# "s-nikon-d90", "s-nikon-d90-2"
assert 0 <= tasks_lens[3] <= 1 #listings with three search tasks: unlikely
#listings that were found by search tasks: "s-nikon-d90" and "s-nikon-d90-2"
assert n_2_d90 == 5
print "prods_lens", prods_lens
assert prods_lens[0] == 0 #listings without an expected product: impossible
assert 8 <= prods_lens[1] <= 10 #listings with one product: D70 or D90
assert 0 <= prods_lens[2] <= 1 #listings with two products: unlikely
assert prods_lens[3] == 0 #listings with three products: impossible
#Test if update has really happened: Only update tasks download descriptions
assert all([d is not None for d in listing["description"]])
print "finished!"
def test_MainObj_create_final_update_tasks():
"""Test DaemonMain.create_final_update_tasks"""
from clair.daemon import DaemonMain
from clair.coredata import Product, SearchTask
conf_dir = relative("..")
data_dir = None
#Create product information and a search task
m = DaemonMain(conf_dir, data_dir)
m.data.set_products([Product("nikon-d90", "Nikon D90", "DSLR Camera",
None, None)])
m.data.add_tasks([SearchTask("nikon-d90", datetime(2000,1,1), "ebay-de",
"Nikon D90", "daily", 30, 10, 500, "EUR",
["nikon-d90"])])
#Execute the search task
m.execute_tasks()
# print m.data.tasks
# print m.data.listings[["title", "price", "sold", "time"]].to_string()
#Create update tasks to get the final price of the listings
m.create_final_update_tasks()
# print m.data.tasks
assert len(m.data.tasks) == 3
assert 0.8 * 30 <= len(m.data.listings) <= 30
#The tasks must be created only once
m.create_final_update_tasks()
assert len(m.data.tasks) == 3
#continuous operation must be possible
m.data.tasks[0].due_time = datetime(2000,1,1)
m.execute_tasks()
m.create_final_update_tasks()
print m.data.tasks
print m.data.listings[["title", "price", "sold", "time"]].to_string()
print "finished!"
