def update_trails(): """ Update trails from feeds """ print datetime.now().strftime('%Y-%m-%d:%H') trails = TrailsDict() trails.update(load_trails()) #load trails list_trails = [] if not (os.path.isfile('trail_info.csv') ): # create a csv for first time of updating print "can't find collecting_info.csv! then init a new csv" old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref']) old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H') old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('trail_info.csv', index=False) else: old = pd.read_csv('trail_info.csv') old_trails = set(old.trail) print "[i] collecting trails information (ftime,ltime)..." filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py"))) filenames = [_ for _ in filenames if "__init__.py" not in _] filenames += sorted(glob.glob(os.path.join( STATIC, "*.py"))) # in static folder, __init__.py has fetch() filenames += sorted(glob.glob(os.path.join( CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch() #remove __init__.py in feeds folder #filenames = [_ for _ in filenames if "__init__.py" not in _] #print filenames init_sys_path() time = datetime.now().strftime('%Y-%m-%d:%H') for i in xrange(len(filenames)): f = filenames[i] try: module = __import__(os.path.basename(f).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[x] Failed: import feed file '%s' ('%s')" % (f, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print "[o] '%s'" % (module.__url__) sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \ (i, len(filenames), i * 100 / len(filenames))) sys.stdout.flush() results = function() for item in results.items(): list_trails.append( (item[0], item[1][0], item[1][1], time, time))
def update_trails(): """ Update trails from feeds """ #check trails folder if not os.path.exists(TRAILS_FOLDER): os.popen("mkdir " + TRAILS_FOLDER) date_now = datetime.now().strftime('%Y-%m-%d') trails_file = date_now + ".csv" #trails_file = os.path.join(TRAILS_FOLDER, date_now) trails_file = os.path.join(TRAILS_FOLDER, trails_file) print trails_file trails = TrailsDict() trails.update(load_trails(trails_file)) #load trails list_trails = [] old_csv = pd.read_csv(trails_file) print "[i] Collecting latest trails ..." filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py"))) filenames += sorted(glob.glob(os.path.join( STATIC, "*.py"))) # in static folder, __init__.py has fetch() filenames += sorted(glob.glob(os.path.join( CUSTOM, "*.py"))) # in custom folder, __init__.py has fetch() #remove __init__.py in feeds folder filenames = [_ for _ in filenames if "__init__.py" not in _] init_sys_path() for i in xrange(len(filenames)): f = filenames[i] try: module = __import__(os.path.basename(f).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[x] Failed: import feed file '%s' ('%s')" % (f, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": try: print "[o] '%s'" % (module.__url__) sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \ (i, len(filenames), i * 100 / len(filenames))) sys.stdout.flush() results = function() for item in results.items(): list_trails.append((item[0], item[1][0], item[1][1])) except Exception, ex: print "[x] Failed: process feed file '%s' ('%s')" % ( filename, ex)
def update_trails(): """ Update trails from feeds """ print datetime.now().strftime('%Y-%m-%d:%H') trails = TrailsDict() trails.update(load_trails()) #load trails '''old=pd.read_csv(TRAILS_CSV,names=['trail', 'info', 'ref']) old['ftime']=datetime.now().strftime('%Y-%m-%d:%H') old['ltime']=datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('old_info.csv',index_label="id") exit(1)''' list_trails = [] if not (os.path.isfile('trail_info.csv') ): # create a csv for first time of updating print "can't find collecting_info.csv!" old = pd.read_csv(TRAILS_CSV, names=['trail', 'info', 'ref']) old['ftime'] = datetime.now().strftime('%Y-%m-%d:%H') old['ltime'] = datetime.now().strftime('%Y-%m-%d:%H') old.to_csv('trail_info.csv', index_label="id") else: old = pd.read_csv('trail_info.csv') old_trails = set(old.trail) print "[i] updating trails (this might take a while)..." filenames = sorted(glob.glob(os.path.join(FEEDS, "*.py"))) filenames += [STATIC] # in static folder, __init__.py has fetch() filenames += [CUSTOM] # in custom folder, __init__.py has fetch() # remove __init__.py in feeds folder filenames = [_ for _ in filenames if "__init__.py" not in _] init_sys_path() for i in xrange(len(filenames)): f = filenames[i] try: module = __import__(os.path.basename(f).split(".py")[0]) except (ImportError, SyntaxError), ex: print "[x] Failed: import feed file '%s' ('%s')" % (f, ex) continue for name, function in inspect.getmembers(module, inspect.isfunction): if name == "fetch": print "[o] '%s'" % (module.__url__) sys.stdout.write("[?] progress: %d/%d (%d%%)\r" % \ (i, len(filenames), i * 100 / len(filenames))) sys.stdout.flush() try: results = function() #print(1) for item in results.items(): list_trails.append( (item[0], item[1][0], item[1][1], datetime.now().strftime('%Y-%m-%d:%H'), datetime.now().strftime('%Y-%m-%d:%H'))) '''if(item[0] in trails): tmp=str(old[old.trail == item[0]].ftime) list_trails.append((item[0], item[1][0],item[1][1],tmp[5:18],time)) #print(1) else: list_trails.append((item[0], item[1][0],item[1][1],time,time)) ''' except Exception, ex: print "[x] Failed: process feed file '%s' ('%s')" % ( filename, ex)