def on_created(self, event): if isinstance(event, DirCreatedEvent): if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) # we only care about this path if it's under a user dir # user/upload/deployment-name path_parts = rel_path.split(os.sep) if len(path_parts) != 3: return logger.info("New deployment directory: %s", rel_path) with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir':event.src_path}) if deployment is None: deployment = db.Deployment() usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } ) if hasattr(usr, '_id'): deployment.user_id = usr._id deployment.name = unicode(path_parts[2]) deployment.deployment_dir = unicode(event.src_path) deployment.updated = datetime.utcnow() deployment.save() elif isinstance(event, FileCreatedEvent): if self.base not in event.src_path: return path_parts = os.path.split(event.src_path) with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]}) if deployment is None: logger.error("Cannot find deployment for %s", path_parts[0]) return if path_parts[-1] == "wmoid.txt": rel_path = os.path.relpath(event.src_path, self.base) logger.info("New wmoid.txt in %s", rel_path) if deployment.wmo_id: logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id) with open(event.src_path) as wf: deployment.wmo_id = unicode(wf.readline().strip()) deployment.save() else: # Always save the Deployment when a new dive file is added # so a checksum is calculated and a new deployment.json file # is created fname, ext = os.path.splitext(path_parts[-1]) if ext not in ['.md5', '.txt']: deployment.save()
def on_created(self, event): if isinstance(event, DirCreatedEvent): if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) # we only care about this path if it's under a user dir # user/upload/deployment-name path_parts = rel_path.split(os.sep) if len(path_parts) != 3: return logger.info("New deployment directory: %s", rel_path) with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir':event.src_path}) if deployment is None: deployment = db.Deployment() usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } ) if hasattr(usr, '_id'): deployment.user_id = usr._id deployment.name = unicode(path_parts[2]) deployment.deployment_dir = unicode(event.src_path) deployment.updated = datetime.utcnow() deployment.save() elif isinstance(event, FileCreatedEvent): if self.base not in event.src_path: return path_parts = os.path.split(event.src_path) if path_parts[-1] != "wmoid.txt": return rel_path = os.path.relpath(event.src_path, self.base) logger.info("New wmoid.txt in %s", rel_path) with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir':path_parts[0]}) if deployment is None: logger.error("Cannot find deployment for %s", path_parts[0]) return if deployment.wmo_id: logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id) with open(event.src_path) as wf: deployment.wmo_id = unicode(wf.readline().strip()) deployment.updated = datetime.utcnow() deployment.save()
def on_created(self, event): if isinstance(event, DirCreatedEvent): if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) # we only care about this path if it's under a user dir # user/upload/deployment-name path_parts = rel_path.split(os.sep) if len(path_parts) != 3: return logger.info("New deployment directory: %s", rel_path) with app.app_context(): deployment = db.Deployment.find_one( {'deployment_dir': event.src_path}) if deployment is None: deployment = db.Deployment() usr = db.User.find_one({'username': str(path_parts[0])}) if hasattr(usr, '_id'): deployment.user_id = usr._id deployment.name = str(path_parts[2]) deployment.deployment_dir = str(event.src_path) deployment.updated = datetime.utcnow() deployment.save() elif isinstance(event, FileCreatedEvent): self.file_moved_or_created(event)
def main(base): with app.app_context(): for uname in users: u = db.User.find_one({'username':uname}) assert u fullpath = os.path.join(base, uname, 'upload') for f in os.listdir(fullpath): if not os.path.isdir(os.path.join(fullpath, f)): continue print "Deployment", f deployment = db.Deployment.find_one({'name':f}) if deployment: print "Found: updating timestamp" deployment.updated = datetime.utcfromtimestamp(os.path.getmtime(os.path.join(fullpath, f))) deployment.save() else: print "Not Found: creating" deployment = db.Deployment() deployment.name = unicode(f) deployment.user_id = u._id deployment.deployment_dir = unicode(os.path.join(fullpath, f)) deployment.completed = os.path.exists(os.path.join(fullpath, f, 'completed.txt')) wmoid_file = os.path.join(fullpath, f, 'wmoid.txt') if os.path.exists(wmoid_file): with open(wmoid_file) as wf: deployment.wmo_id = unicode(wf.readline().strip()) deployment.save()
def update_db(src, dest): with app.app_context(): provider, deployment_name = deployment_split(src) d_provider, d_deployment_name = deployment_split(dest) deployments = list(db.Deployment.find({'name': deployment_name})) for dep in deployments: print dep.name, '->', d_deployment_name dep.name = unicode(d_deployment_name) print dep.deployment_dir, '->', dest dep.deployment_dir = unicode(dest) dep.save()
def update_db(src, dest): with app.app_context(): provider, deployment_name = deployment_split(src) d_provider, d_deployment_name = deployment_split(dest) deployments = list(db.Deployment.find({'name':deployment_name})) for dep in deployments: print dep.name, '->', d_deployment_name dep.name = unicode(d_deployment_name) print dep.deployment_dir, '->', dest dep.deployment_dir = unicode(dest) dep.save()
def file_moved_or_created(self, event): logger.info('%s %s', self.base, event.src_path) if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) if isinstance(event, FileMovedEvent): rel_path = os.path.relpath(event.dest_path, self.base) path_parts = os.path.split(rel_path) logger.info("%s %s", type(event), path_parts) # ignore if a dotfile if path_parts[1].startswith('.'): return with app.app_context(): deployment = db.Deployment.find_one( {'deployment_dir': path_parts[0]}) if deployment is None: logger.error("Cannot find deployment for %s", path_parts[0]) return if path_parts[-1] == "wmoid.txt": rel_path = os.path.relpath(event.src_path, self.base) logger.info("New wmoid.txt in %s", rel_path) if deployment.wmo_id: logger.info( "Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id) with open(event.src_path) as wf: deployment.wmo_id = str(wf.readline().strip()) deployment.save() logger.info("Updated deployment %s", path_parts[0]) else: # Always save the Deployment when a new dive file is added # so a checksum is calculated and a new deployment.json file # is created fname, ext = os.path.splitext(path_parts[-1]) if '.nc' in ext: deployment.save() logger.info("Updated deployment %s", path_parts[0]) # touch the ERDDAP flag (realtime data only) if not deployment.delayed_mode: deployment_name = path_parts[0].split('/')[-1] self.touch_erddap(deployment_name)
def on_deleted(self, event): if isinstance(event, DirDeletedEvent): if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) # we only care about this path if it's under a user dir # user/upload/deployment-name path_parts = rel_path.split(os.sep) if len(path_parts) != 3: return logger.info("Removed deployment directory: %s", rel_path) with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir':event.src_path}) if deployment: deployment.delete()
def main(base): with app.app_context(): for uname in users: u = db.User.find_one({'username': uname}) assert u fullpath = os.path.join(base, uname, 'upload') for f in os.listdir(fullpath): if not os.path.isdir(os.path.join(fullpath, f)): continue print "Deployment", f deployment = db.Deployment.find_one({'name': f}) if deployment: print "Found: updating timestamp" deployment.updated = datetime.utcfromtimestamp( os.path.getmtime(os.path.join(fullpath, f))) deployment.save() else: print "Not Found: creating" deployment = db.Deployment() deployment.name = unicode(f) deployment.user_id = u._id deployment.deployment_dir = unicode( os.path.join(fullpath, f)) deployment.completed = os.path.exists( os.path.join(fullpath, f, 'completed.txt')) wmoid_file = os.path.join(fullpath, f, 'wmoid.txt') if os.path.exists(wmoid_file): with open(wmoid_file) as wf: deployment.wmo_id = unicode(wf.readline().strip()) deployment.save()
def file_moved_or_created(self, event): logger.info('%s %s', self.base, event.src_path) if self.base not in event.src_path: return rel_path = os.path.relpath(event.src_path, self.base) path_parts = os.path.split(rel_path) logger.info("%s %s", type(event), path_parts) # ignore if a dotfile if path_parts[1].startswith('.'): return with app.app_context(): deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]}) if deployment is None: logger.error("Cannot find deployment for %s", path_parts[0]) return if path_parts[-1] == "wmoid.txt": rel_path = os.path.relpath(event.src_path, self.base) logger.info("New wmoid.txt in %s", rel_path) if deployment.wmo_id: logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id) with open(event.src_path) as wf: deployment.wmo_id = unicode(wf.readline().strip()) deployment.save() logger.info("Updated deployment %s", path_parts[0]) else: # Always save the Deployment when a new dive file is added # so a checksum is calculated and a new deployment.json file # is created fname, ext = os.path.splitext(path_parts[-1]) if ext not in ['.md5', '.txt']: deployment.save() logger.info("Updated deployment %s", path_parts[0])
#!/usr/bin/env python ''' scripts/update_deployments.py A script to update (save) all the deployments in mongo (Deployment model) It can be used to properly update all the old deployments latest_file and latest_file_mtime attributes. Those attributes are updated in the .save method of the Deployment model. This is not a script that should be run on a schedule. Only one-offs as necessary from within the container. ''' from glider_dac import app, db if __name__ == '__main__': with app.app_context(): deployments = db.Deployment.find() for d in deployments: d.save()
''' Gets the best candidate for a WMO ID :param netCDF4.Dataset nc: An open netCDF4 Dataset ''' if getattr(nc, 'wmo_id', None): wmo_id = nc.wmo_id elif 'wmo_id' in nc.variables: wmo_id = ''.join(nc.variables['wmo_id'][:].flatten()) else: return None return wmo_id def get_acknowledgment(nc): ''' Gets the best candidate for an acknowledgment :param netCDF4.Dataset nc: An open netCDF4 Dataset ''' return getattr(nc, 'acknowledgment', None) if __name__ == '__main__': from argparse import ArgumentParser parser = ArgumentParser(description=main.__doc__) args = parser.parse_args() with app.app_context(): sys.exit(main(args))
def glider_deployment_check(data_type=None, completed=True, force=False, deployment_dir=None, username=None): """ """ cs = CheckSuite() cs.load_all_available_checkers() with app.app_context(): if data_type is not None: is_delayed_mode = data_type == 'delayed' if is_delayed_mode: q_dict = {"delayed_mode": True, "completed": completed} else: q_dict = { "$or": [{ "delayed_mode": False }, { "delayed_mode": { "$exists": False } }], "completed": completed } if not force: q_dict["compliance_check_passed"] = {"$ne": True} # TODO: combine username/deployment cases? if username: q_dict = {"username": username} # a particular deployment has been specified elif deployment_dir: q_dict = {"deployment_dir": deployment_dir} else: q_dict = {} agg_pipeline = [{ "$match": q_dict }, { "$group": { "_id": "$user_id", "deployments": { "$push": { "_id": "$_id", "name": "$name", "deployment_dir": "$deployment_dir" } } } }] # if force is enabled, re-check the datasets no matter what # is this syntax still used? if the first fn call fails, use the # second set of results try: agg_result_set = db.deployments.aggregate(agg_pipeline)['result'] except: agg_result_set = db.deployments.aggregate(agg_pipeline, cursor={}) for res in agg_result_set: user = db.users.find_one(res["_id"]) all_messages = [] failing_deployments = [] for dep in res['deployments']: root_logger.info("Running compliance check on glider " "deployment: {}".format(dep)) try: dep_passed, dep_messages = process_deployment(dep) all_messages.append(dep_messages) if not dep_passed: failing_deployments.append(dep) except Exception as e: root_logger.exception( "Exception occurred while processing deployment {}". format(dep['name'])) text_body = '' send_deployment_cchecker_email(user, failing_deployments, "\n".join(all_messages))
def main(): with app.app_context(): update()
def wrapper(*args, **kwargs): with app.app_context(): return f(*args, **kwargs)