def update_trails():
"""
Update trails from feeds
"""
print datetime.now().strftime('%Y-%m-%d:%H')
trails = TrailsDict()
trails.update(load_trails()) #load trails
list_trails = []
if not (os.path.isfile('trail_info.csv')
): # create a csv for first time of updating
print "can't find collecting_info.csv! then init a new csv"
old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref'])
old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H')
old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('trail_info.csv', index=False)
else:
old = pd.read_csv('trail_info.csv')
old_trails = set(old.trail)
print "[i] collecting trails information (ftime,ltime)..."
filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py")))
filenames = [_ for _ in filenames if "__init__.py" not in _]
filenames += sorted(glob.glob(os.path.join(
STATIC, "*.py"))) # in static folder, __init__.py has fetch()
filenames += sorted(glob.glob(os.path.join(
CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch()
#remove __init__.py in feeds folder
#filenames = [_ for _ in filenames if "__init__.py" not in _]
#print filenames
init_sys_path()
time = datetime.now().strftime('%Y-%m-%d:%H')
for i in xrange(len(filenames)):
f = filenames[i]
try:
module = __import__(os.path.basename(f).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[x] Failed: import feed file '%s' ('%s')" % (f, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print "[o] '%s'" % (module.__url__)
sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \
(i, len(filenames), i * 100 / len(filenames)))
sys.stdout.flush()
results = function()
for item in results.items():
list_trails.append(
(item[0], item[1][0], item[1][1], time, time))
def update_trails():
"""
Update trails from feeds
"""
#check trails folder
if not os.path.exists(TRAILS_FOLDER):
os.popen("mkdir " + TRAILS_FOLDER)
date_now = datetime.now().strftime('%Y-%m-%d')
trails_file = date_now + ".csv"
#trails_file = os.path.join(TRAILS_FOLDER, date_now)
trails_file = os.path.join(TRAILS_FOLDER, trails_file)
print trails_file
trails = TrailsDict()
trails.update(load_trails(trails_file)) #load trails
list_trails = []
old_csv = pd.read_csv(trails_file)
print "[i] Collecting latest trails ..."
filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py")))
filenames += sorted(glob.glob(os.path.join(
STATIC, "*.py"))) # in static folder, __init__.py has fetch()
filenames += sorted(glob.glob(os.path.join(
CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch()
#remove __init__.py in feeds folder
filenames = [_ for _ in filenames if "__init__.py" not in _]
init_sys_path()
for i in xrange(len(filenames)):
f = filenames[i]
try:
module = __import__(os.path.basename(f).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[x] Failed: import feed file '%s' ('%s')" % (f, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
try:
print "[o] '%s'" % (module.__url__)
sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \
(i, len(filenames), i * 100 / len(filenames)))
sys.stdout.flush()
results = function()
for item in results.items():
list_trails.append((item[0], item[1][0], item[1][1]))
except Exception, ex:
print "[x] Failed: process feed file '%s' ('%s')" % (
filename, ex)
def update_trails():
"""
Update trails from feeds
"""
print datetime.now().strftime('%Y-%m-%d:%H')
trails = TrailsDict()
trails.update(load_trails()) #load trails
'''old=pd.read_csv(TRAILS_CSV,names=['trail', 'info', 'ref'])
old['ftime']=datetime.now().strftime('%Y-%m-%d:%H')
old['ltime']=datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('old_info.csv',index_label="id")
exit(1)'''
list_trails = []
if not (os.path.isfile('trail_info.csv')
): # create a csv for first time of updating
print "can't find collecting_info.csv!"
old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref'])
old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H')
old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H')
old.to_csv('trail_info.csv', index_label="id")
else:
old = pd.read_csv('trail_info.csv')
old_trails = set(old.trail)
print "[i] updating trails (this might take a while)..."
filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py")))
filenames += [STATIC] # in static folder, __init__.py has fetch()
filenames += [CUSTOM] # in custom folder, __init__.py has fetch()
# remove __init__.py in feeds folder
filenames = [_ for _ in filenames if "__init__.py" not in _]
init_sys_path()
for i in xrange(len(filenames)):
f = filenames[i]
try:
module = __import__(os.path.basename(f).split(".py")[0])
except (ImportError, SyntaxError), ex:
print "[x] Failed: import feed file '%s' ('%s')" % (f, ex)
continue
for name, function in inspect.getmembers(module, inspect.isfunction):
if name == "fetch":
print "[o] '%s'" % (module.__url__)
sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \
(i, len(filenames), i * 100 / len(filenames)))
sys.stdout.flush()
try:
results = function()
#print(1)
for item in results.items():
list_trails.append(
(item[0], item[1][0], item[1][1],
datetime.now().strftime('%Y-%m-%d:%H'),
datetime.now().strftime('%Y-%m-%d:%H')))
'''if(item[0] in trails):
tmp=str(old[old.trail == item[0]].ftime)
list_trails.append((item[0], item[1][0],item[1][1],tmp[5:18],time))
#print(1)
else:
list_trails.append((item[0], item[1][0],item[1][1],time,time))
'''
except Exception, ex:
print "[x] Failed: process feed file '%s' ('%s')" % (
filename, ex)
