def on_created(self, event):
if isinstance(event, DirCreatedEvent):
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
# we only care about this path if it's under a user dir
# user/upload/deployment-name
path_parts = rel_path.split(os.sep)
if len(path_parts) != 3:
return
logger.info("New deployment directory: %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
if deployment is None:
deployment = db.Deployment()
usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } )
if hasattr(usr, '_id'):
deployment.user_id = usr._id
deployment.name = unicode(path_parts[2])
deployment.deployment_dir = unicode(event.src_path)
deployment.updated = datetime.utcnow()
deployment.save()
elif isinstance(event, FileCreatedEvent):
if self.base not in event.src_path:
return
path_parts = os.path.split(event.src_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]})
if deployment is None:
logger.error("Cannot find deployment for %s", path_parts[0])
return
if path_parts[-1] == "wmoid.txt":
rel_path = os.path.relpath(event.src_path, self.base)
logger.info("New wmoid.txt in %s", rel_path)
if deployment.wmo_id:
logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id)
with open(event.src_path) as wf:
deployment.wmo_id = unicode(wf.readline().strip())
deployment.save()
else:
# Always save the Deployment when a new dive file is added
# so a checksum is calculated and a new deployment.json file
# is created
fname, ext = os.path.splitext(path_parts[-1])
if ext not in ['.md5', '.txt']:
deployment.save()
def on_created(self, event):
if isinstance(event, DirCreatedEvent):
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
# we only care about this path if it's under a user dir
# user/upload/deployment-name
path_parts = rel_path.split(os.sep)
if len(path_parts) != 3:
return
logger.info("New deployment directory: %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
if deployment is None:
deployment = db.Deployment()
usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } )
if hasattr(usr, '_id'):
deployment.user_id = usr._id
deployment.name = unicode(path_parts[2])
deployment.deployment_dir = unicode(event.src_path)
deployment.updated = datetime.utcnow()
deployment.save()
elif isinstance(event, FileCreatedEvent):
if self.base not in event.src_path:
return
path_parts = os.path.split(event.src_path)
if path_parts[-1] != "wmoid.txt":
return
rel_path = os.path.relpath(event.src_path, self.base)
logger.info("New wmoid.txt in %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir':path_parts[0]})
if deployment is None:
logger.error("Cannot find deployment for %s", path_parts[0])
return
if deployment.wmo_id:
logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id)
with open(event.src_path) as wf:
deployment.wmo_id = unicode(wf.readline().strip())
deployment.updated = datetime.utcnow()
deployment.save()
def on_created(self, event):
if isinstance(event, DirCreatedEvent):
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
# we only care about this path if it's under a user dir
# user/upload/deployment-name
path_parts = rel_path.split(os.sep)
if len(path_parts) != 3:
return
logger.info("New deployment directory: %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one(
{'deployment_dir': event.src_path})
if deployment is None:
deployment = db.Deployment()
usr = db.User.find_one({'username': str(path_parts[0])})
if hasattr(usr, '_id'):
deployment.user_id = usr._id
deployment.name = str(path_parts[2])
deployment.deployment_dir = str(event.src_path)
deployment.updated = datetime.utcnow()
deployment.save()
elif isinstance(event, FileCreatedEvent):
self.file_moved_or_created(event)
def main(base):
with app.app_context():
for uname in users:
u = db.User.find_one({'username':uname})
assert u
fullpath = os.path.join(base, uname, 'upload')
for f in os.listdir(fullpath):
if not os.path.isdir(os.path.join(fullpath, f)):
continue
print "Deployment", f
deployment = db.Deployment.find_one({'name':f})
if deployment:
print "Found: updating timestamp"
deployment.updated = datetime.utcfromtimestamp(os.path.getmtime(os.path.join(fullpath, f)))
deployment.save()
else:
print "Not Found: creating"
deployment = db.Deployment()
deployment.name = unicode(f)
deployment.user_id = u._id
deployment.deployment_dir = unicode(os.path.join(fullpath, f))
deployment.completed = os.path.exists(os.path.join(fullpath, f, 'completed.txt'))
wmoid_file = os.path.join(fullpath, f, 'wmoid.txt')
if os.path.exists(wmoid_file):
with open(wmoid_file) as wf:
deployment.wmo_id = unicode(wf.readline().strip())
deployment.save()
def update_db(src, dest):
with app.app_context():
provider, deployment_name = deployment_split(src)
d_provider, d_deployment_name = deployment_split(dest)
deployments = list(db.Deployment.find({'name': deployment_name}))
for dep in deployments:
print dep.name, '->', d_deployment_name
dep.name = unicode(d_deployment_name)
print dep.deployment_dir, '->', dest
dep.deployment_dir = unicode(dest)
dep.save()
def update_db(src, dest):
with app.app_context():
provider, deployment_name = deployment_split(src)
d_provider, d_deployment_name = deployment_split(dest)
deployments = list(db.Deployment.find({'name':deployment_name}))
for dep in deployments:
print dep.name, '->', d_deployment_name
dep.name = unicode(d_deployment_name)
print dep.deployment_dir, '->', dest
dep.deployment_dir = unicode(dest)
dep.save()
def file_moved_or_created(self, event):
logger.info('%s %s', self.base, event.src_path)
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
if isinstance(event, FileMovedEvent):
rel_path = os.path.relpath(event.dest_path, self.base)
path_parts = os.path.split(rel_path)
logger.info("%s %s", type(event), path_parts)
# ignore if a dotfile
if path_parts[1].startswith('.'):
return
with app.app_context():
deployment = db.Deployment.find_one(
{'deployment_dir': path_parts[0]})
if deployment is None:
logger.error("Cannot find deployment for %s", path_parts[0])
return
if path_parts[-1] == "wmoid.txt":
rel_path = os.path.relpath(event.src_path, self.base)
logger.info("New wmoid.txt in %s", rel_path)
if deployment.wmo_id:
logger.info(
"Deployment already has wmoid %s. Updating value with new file.",
deployment.wmo_id)
with open(event.src_path) as wf:
deployment.wmo_id = str(wf.readline().strip())
deployment.save()
logger.info("Updated deployment %s", path_parts[0])
else:
# Always save the Deployment when a new dive file is added
# so a checksum is calculated and a new deployment.json file
# is created
fname, ext = os.path.splitext(path_parts[-1])
if '.nc' in ext:
deployment.save()
logger.info("Updated deployment %s", path_parts[0])
# touch the ERDDAP flag (realtime data only)
if not deployment.delayed_mode:
deployment_name = path_parts[0].split('/')[-1]
self.touch_erddap(deployment_name)
def on_deleted(self, event):
if isinstance(event, DirDeletedEvent):
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
# we only care about this path if it's under a user dir
# user/upload/deployment-name
path_parts = rel_path.split(os.sep)
if len(path_parts) != 3:
return
logger.info("Removed deployment directory: %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
if deployment:
deployment.delete()
def on_deleted(self, event):
if isinstance(event, DirDeletedEvent):
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
# we only care about this path if it's under a user dir
# user/upload/deployment-name
path_parts = rel_path.split(os.sep)
if len(path_parts) != 3:
return
logger.info("Removed deployment directory: %s", rel_path)
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
if deployment:
deployment.delete()
def main(base):
with app.app_context():
for uname in users:
u = db.User.find_one({'username': uname})
assert u
fullpath = os.path.join(base, uname, 'upload')
for f in os.listdir(fullpath):
if not os.path.isdir(os.path.join(fullpath, f)):
continue
print "Deployment", f
deployment = db.Deployment.find_one({'name': f})
if deployment:
print "Found: updating timestamp"
deployment.updated = datetime.utcfromtimestamp(
os.path.getmtime(os.path.join(fullpath, f)))
deployment.save()
else:
print "Not Found: creating"
deployment = db.Deployment()
deployment.name = unicode(f)
deployment.user_id = u._id
deployment.deployment_dir = unicode(
os.path.join(fullpath, f))
deployment.completed = os.path.exists(
os.path.join(fullpath, f, 'completed.txt'))
wmoid_file = os.path.join(fullpath, f, 'wmoid.txt')
if os.path.exists(wmoid_file):
with open(wmoid_file) as wf:
deployment.wmo_id = unicode(wf.readline().strip())
deployment.save()
def file_moved_or_created(self, event):
logger.info('%s %s', self.base, event.src_path)
if self.base not in event.src_path:
return
rel_path = os.path.relpath(event.src_path, self.base)
path_parts = os.path.split(rel_path)
logger.info("%s %s", type(event), path_parts)
# ignore if a dotfile
if path_parts[1].startswith('.'):
return
with app.app_context():
deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]})
if deployment is None:
logger.error("Cannot find deployment for %s", path_parts[0])
return
if path_parts[-1] == "wmoid.txt":
rel_path = os.path.relpath(event.src_path, self.base)
logger.info("New wmoid.txt in %s", rel_path)
if deployment.wmo_id:
logger.info("Deployment already has wmoid %s. Updating value with new file.", deployment.wmo_id)
with open(event.src_path) as wf:
deployment.wmo_id = unicode(wf.readline().strip())
deployment.save()
logger.info("Updated deployment %s", path_parts[0])
else:
# Always save the Deployment when a new dive file is added
# so a checksum is calculated and a new deployment.json file
# is created
fname, ext = os.path.splitext(path_parts[-1])
if ext not in ['.md5', '.txt']:
deployment.save()
logger.info("Updated deployment %s", path_parts[0])
#!/usr/bin/env python
'''
scripts/update_deployments.py
A script to update (save) all the deployments in mongo (Deployment model)
It can be used to properly update all the old deployments latest_file and
latest_file_mtime attributes. Those attributes are updated in the .save
method of the Deployment model.
This is not a script that should be run on a schedule. Only one-offs as
necessary from within the container.
'''
from glider_dac import app, db
if __name__ == '__main__':
with app.app_context():
deployments = db.Deployment.find()
for d in deployments:
d.save()
'''
Gets the best candidate for a WMO ID
:param netCDF4.Dataset nc: An open netCDF4 Dataset
'''
if getattr(nc, 'wmo_id', None):
wmo_id = nc.wmo_id
elif 'wmo_id' in nc.variables:
wmo_id = ''.join(nc.variables['wmo_id'][:].flatten())
else:
return None
return wmo_id
def get_acknowledgment(nc):
'''
Gets the best candidate for an acknowledgment
:param netCDF4.Dataset nc: An open netCDF4 Dataset
'''
return getattr(nc, 'acknowledgment', None)
if __name__ == '__main__':
from argparse import ArgumentParser
parser = ArgumentParser(description=main.__doc__)
args = parser.parse_args()
with app.app_context():
sys.exit(main(args))
def glider_deployment_check(data_type=None,
completed=True,
force=False,
deployment_dir=None,
username=None):
"""
"""
cs = CheckSuite()
cs.load_all_available_checkers()
with app.app_context():
if data_type is not None:
is_delayed_mode = data_type == 'delayed'
if is_delayed_mode:
q_dict = {"delayed_mode": True, "completed": completed}
else:
q_dict = {
"$or": [{
"delayed_mode": False
}, {
"delayed_mode": {
"$exists": False
}
}],
"completed":
completed
}
if not force:
q_dict["compliance_check_passed"] = {"$ne": True}
# TODO: combine username/deployment cases?
if username:
q_dict = {"username": username}
# a particular deployment has been specified
elif deployment_dir:
q_dict = {"deployment_dir": deployment_dir}
else:
q_dict = {}
agg_pipeline = [{
"$match": q_dict
}, {
"$group": {
"_id": "$user_id",
"deployments": {
"$push": {
"_id": "$_id",
"name": "$name",
"deployment_dir": "$deployment_dir"
}
}
}
}]
# if force is enabled, re-check the datasets no matter what
# is this syntax still used? if the first fn call fails, use the
# second set of results
try:
agg_result_set = db.deployments.aggregate(agg_pipeline)['result']
except:
agg_result_set = db.deployments.aggregate(agg_pipeline, cursor={})
for res in agg_result_set:
user = db.users.find_one(res["_id"])
all_messages = []
failing_deployments = []
for dep in res['deployments']:
root_logger.info("Running compliance check on glider "
"deployment: {}".format(dep))
try:
dep_passed, dep_messages = process_deployment(dep)
all_messages.append(dep_messages)
if not dep_passed:
failing_deployments.append(dep)
except Exception as e:
root_logger.exception(
"Exception occurred while processing deployment {}".
format(dep['name']))
text_body = ''
send_deployment_cchecker_email(user, failing_deployments,
"\n".join(all_messages))
def main():
with app.app_context():
update()
def main():
with app.app_context():
update()
def wrapper(*args, **kwargs):
with app.app_context():
return f(*args, **kwargs)
