def updateAssociatedFiles(sources,collection): for source in sources: if IsDir(source + '__FILES__'): G = gfs.GridFS(collection.database,collection=collection.name) for file in listdir(source + '__FILES__'): os.environ['PROTECTION'] = 'OFF' S = open(source + '__FILES__/' + file,'r').read() os.environ['PROTECTION'] = 'ON' G.put(S,filename = file)
def download_check(download_dir, incremental, certpath): if not incremental: check_list = [download_dir ] else: check_list = get_increment_paths(download_dir) assert all(['__PARSE__' in listdir(p) for p in check_list]) createCertificate(certpath,'Collection properly downloaded and pre-parsed.')
def get_and_check_increments(download_dir): X = re.compile('__[\d]+__$') L = [x for x in listdir(download_dir) if not x.startswith('.')] assert all([X.match(x) for x in L]) return [int(x[2:-2]) for x in L]