def updateAssociatedFiles(sources,collection):
for source in sources:
if IsDir(source + '__FILES__'):
G = gfs.GridFS(collection.database,collection=collection.name)
for file in listdir(source + '__FILES__'):
os.environ['PROTECTION'] = 'OFF'
S = open(source + '__FILES__/' + file,'r').read()
os.environ['PROTECTION'] = 'ON'
G.put(S,filename = file)
def download_check(download_dir, incremental, certpath):
if not incremental:
check_list = [download_dir ]
else:
check_list = get_increment_paths(download_dir)
assert all(['__PARSE__' in listdir(p) for p in check_list])
createCertificate(certpath,'Collection properly downloaded and pre-parsed.')
def get_and_check_increments(download_dir):
X = re.compile('__[\d]+__$')
L = [x for x in listdir(download_dir) if not x.startswith('.')]
assert all([X.match(x) for x in L])
return [int(x[2:-2]) for x in L]
