Ejemplo n.º 1
0
    def on_created(self, event):
        if isinstance(event, DirCreatedEvent):

            if self.base not in event.src_path:
                return

            rel_path = os.path.relpath(event.src_path, self.base)

            # we only care about this path if it's under a user dir
            # user/upload/deployment-name
            path_parts = rel_path.split(os.sep)

            if len(path_parts) != 3:
                return

            logger.info("New deployment directory: %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
                if deployment is None:
                    deployment             = db.Deployment()

                    usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } )
                    if hasattr(usr, '_id'):
                        deployment.user_id     = usr._id
                        deployment.name        = unicode(path_parts[2])
                        deployment.deployment_dir = unicode(event.src_path)
                        deployment.updated     = datetime.utcnow()
                        deployment.save()

        elif isinstance(event, FileCreatedEvent):
            if self.base not in event.src_path:
                return

            path_parts = os.path.split(event.src_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]})
                if deployment is None:
                    logger.error("Cannot find deployment for %s", path_parts[0])
                    return

                if path_parts[-1] == "wmoid.txt":
                    rel_path = os.path.relpath(event.src_path, self.base)
                    logger.info("New wmoid.txt in %s", rel_path)
                    if deployment.wmo_id:
                        logger.info("Deployment already has wmoid %s.  Updating value with new file.", deployment.wmo_id)
                    with open(event.src_path) as wf:
                        deployment.wmo_id = unicode(wf.readline().strip())
                    deployment.save()
                else:
                    # Always save the Deployment when a new dive file is added
                    # so a checksum is calculated and a new deployment.json file
                    # is created
                    fname, ext = os.path.splitext(path_parts[-1])
                    if ext not in ['.md5', '.txt']:
                        deployment.save()
Ejemplo n.º 2
0
    def on_created(self, event):
        if isinstance(event, DirCreatedEvent):

            if self.base not in event.src_path:
                return

            rel_path = os.path.relpath(event.src_path, self.base)

            # we only care about this path if it's under a user dir
            # user/upload/deployment-name
            path_parts = rel_path.split(os.sep)

            if len(path_parts) != 3:
                return

            logger.info("New deployment directory: %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
                if deployment is None:
                    deployment             = db.Deployment()

                    usr = db.User.find_one( { 'username' : unicode(path_parts[0]) } )
                    if hasattr(usr, '_id'):
                        deployment.user_id     = usr._id
                        deployment.name        = unicode(path_parts[2])
                        deployment.deployment_dir = unicode(event.src_path)
                        deployment.updated     = datetime.utcnow()
                        deployment.save()

        elif isinstance(event, FileCreatedEvent):
            if self.base not in event.src_path:
                return

            path_parts = os.path.split(event.src_path)

            if path_parts[-1] != "wmoid.txt":
                return

            rel_path = os.path.relpath(event.src_path, self.base)
            logger.info("New wmoid.txt in %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir':path_parts[0]})
                if deployment is None:
                    logger.error("Cannot find deployment for %s", path_parts[0])
                    return

                if deployment.wmo_id:
                    logger.info("Deployment already has wmoid %s.  Updating value with new file.", deployment.wmo_id)

                with open(event.src_path) as wf:
                    deployment.wmo_id = unicode(wf.readline().strip())

                deployment.updated     = datetime.utcnow()
                deployment.save()
Ejemplo n.º 3
0
    def on_created(self, event):
        if isinstance(event, DirCreatedEvent):

            if self.base not in event.src_path:
                return

            rel_path = os.path.relpath(event.src_path, self.base)

            # we only care about this path if it's under a user dir
            # user/upload/deployment-name
            path_parts = rel_path.split(os.sep)

            if len(path_parts) != 3:
                return

            logger.info("New deployment directory: %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one(
                    {'deployment_dir': event.src_path})
                if deployment is None:
                    deployment = db.Deployment()

                    usr = db.User.find_one({'username': str(path_parts[0])})
                    if hasattr(usr, '_id'):
                        deployment.user_id = usr._id
                        deployment.name = str(path_parts[2])
                        deployment.deployment_dir = str(event.src_path)
                        deployment.updated = datetime.utcnow()
                        deployment.save()

        elif isinstance(event, FileCreatedEvent):
            self.file_moved_or_created(event)
Ejemplo n.º 4
0
def main(base):
    with app.app_context():
        for uname in users:
            u = db.User.find_one({'username':uname})
            assert u

            fullpath = os.path.join(base, uname, 'upload')

            for f in os.listdir(fullpath):
                if not os.path.isdir(os.path.join(fullpath, f)):
                    continue

                print "Deployment", f

                deployment = db.Deployment.find_one({'name':f})

                if deployment:
                    print "Found: updating timestamp"
                    deployment.updated = datetime.utcfromtimestamp(os.path.getmtime(os.path.join(fullpath, f)))
                    deployment.save()
                else:
                    print "Not Found: creating"
                    deployment = db.Deployment()
                    deployment.name = unicode(f)
                    deployment.user_id = u._id
                    deployment.deployment_dir = unicode(os.path.join(fullpath, f))

                    deployment.completed = os.path.exists(os.path.join(fullpath, f, 'completed.txt'))

                    wmoid_file = os.path.join(fullpath, f, 'wmoid.txt')
                    if os.path.exists(wmoid_file):
                        with open(wmoid_file) as wf:
                            deployment.wmo_id = unicode(wf.readline().strip())

                    deployment.save()
Ejemplo n.º 5
0
def update_db(src, dest):
    with app.app_context():
        provider, deployment_name = deployment_split(src)
        d_provider, d_deployment_name = deployment_split(dest)
        deployments = list(db.Deployment.find({'name': deployment_name}))
        for dep in deployments:
            print dep.name, '->', d_deployment_name
            dep.name = unicode(d_deployment_name)
            print dep.deployment_dir, '->', dest
            dep.deployment_dir = unicode(dest)
            dep.save()
Ejemplo n.º 6
0
def update_db(src, dest):
    with app.app_context():
        provider, deployment_name = deployment_split(src)
        d_provider, d_deployment_name = deployment_split(dest)
        deployments = list(db.Deployment.find({'name':deployment_name}))
        for dep in deployments:
            print dep.name, '->', d_deployment_name
            dep.name = unicode(d_deployment_name)
            print dep.deployment_dir, '->', dest
            dep.deployment_dir = unicode(dest)
            dep.save()
Ejemplo n.º 7
0
    def file_moved_or_created(self, event):
        logger.info('%s %s', self.base, event.src_path)
        if self.base not in event.src_path:
            return

        rel_path = os.path.relpath(event.src_path, self.base)
        if isinstance(event, FileMovedEvent):
            rel_path = os.path.relpath(event.dest_path, self.base)
        path_parts = os.path.split(rel_path)
        logger.info("%s %s", type(event), path_parts)

        # ignore if a dotfile
        if path_parts[1].startswith('.'):
            return

        with app.app_context():
            deployment = db.Deployment.find_one(
                {'deployment_dir': path_parts[0]})
            if deployment is None:
                logger.error("Cannot find deployment for %s", path_parts[0])
                return

            if path_parts[-1] == "wmoid.txt":
                rel_path = os.path.relpath(event.src_path, self.base)
                logger.info("New wmoid.txt in %s", rel_path)
                if deployment.wmo_id:
                    logger.info(
                        "Deployment already has wmoid %s.  Updating value with new file.",
                        deployment.wmo_id)
                with open(event.src_path) as wf:
                    deployment.wmo_id = str(wf.readline().strip())
                deployment.save()
                logger.info("Updated deployment %s", path_parts[0])
            else:
                # Always save the Deployment when a new dive file is added
                # so a checksum is calculated and a new deployment.json file
                # is created
                fname, ext = os.path.splitext(path_parts[-1])
                if '.nc' in ext:
                    deployment.save()
                    logger.info("Updated deployment %s", path_parts[0])
                    # touch the ERDDAP flag (realtime data only)
                    if not deployment.delayed_mode:
                        deployment_name = path_parts[0].split('/')[-1]
                        self.touch_erddap(deployment_name)
Ejemplo n.º 8
0
    def on_deleted(self, event):
        if isinstance(event, DirDeletedEvent):
            if self.base not in event.src_path:
                return

            rel_path = os.path.relpath(event.src_path, self.base)

            # we only care about this path if it's under a user dir
            # user/upload/deployment-name
            path_parts = rel_path.split(os.sep)

            if len(path_parts) != 3:
                return

            logger.info("Removed deployment directory: %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
                if deployment:
                    deployment.delete()
Ejemplo n.º 9
0
    def on_deleted(self, event):
        if isinstance(event, DirDeletedEvent):
            if self.base not in event.src_path:
                return

            rel_path = os.path.relpath(event.src_path, self.base)

            # we only care about this path if it's under a user dir
            # user/upload/deployment-name
            path_parts = rel_path.split(os.sep)

            if len(path_parts) != 3:
                return

            logger.info("Removed deployment directory: %s", rel_path)

            with app.app_context():
                deployment = db.Deployment.find_one({'deployment_dir':event.src_path})
                if deployment:
                    deployment.delete()
Ejemplo n.º 10
0
def main(base):
    with app.app_context():
        for uname in users:
            u = db.User.find_one({'username': uname})
            assert u

            fullpath = os.path.join(base, uname, 'upload')

            for f in os.listdir(fullpath):
                if not os.path.isdir(os.path.join(fullpath, f)):
                    continue

                print "Deployment", f

                deployment = db.Deployment.find_one({'name': f})

                if deployment:
                    print "Found: updating timestamp"
                    deployment.updated = datetime.utcfromtimestamp(
                        os.path.getmtime(os.path.join(fullpath, f)))
                    deployment.save()
                else:
                    print "Not Found: creating"
                    deployment = db.Deployment()
                    deployment.name = unicode(f)
                    deployment.user_id = u._id
                    deployment.deployment_dir = unicode(
                        os.path.join(fullpath, f))

                    deployment.completed = os.path.exists(
                        os.path.join(fullpath, f, 'completed.txt'))

                    wmoid_file = os.path.join(fullpath, f, 'wmoid.txt')
                    if os.path.exists(wmoid_file):
                        with open(wmoid_file) as wf:
                            deployment.wmo_id = unicode(wf.readline().strip())

                    deployment.save()
Ejemplo n.º 11
0
    def file_moved_or_created(self, event):
        logger.info('%s %s', self.base, event.src_path)
        if self.base not in event.src_path:
            return

        rel_path = os.path.relpath(event.src_path, self.base)
        path_parts = os.path.split(rel_path)
        logger.info("%s %s", type(event), path_parts)

        # ignore if a dotfile
        if path_parts[1].startswith('.'):
            return


        with app.app_context():
            deployment = db.Deployment.find_one({'deployment_dir' : path_parts[0]})
            if deployment is None:
                logger.error("Cannot find deployment for %s", path_parts[0])
                return

            if path_parts[-1] == "wmoid.txt":
                rel_path = os.path.relpath(event.src_path, self.base)
                logger.info("New wmoid.txt in %s", rel_path)
                if deployment.wmo_id:
                    logger.info("Deployment already has wmoid %s.  Updating value with new file.", deployment.wmo_id)
                with open(event.src_path) as wf:
                    deployment.wmo_id = unicode(wf.readline().strip())
                deployment.save()
                logger.info("Updated deployment %s", path_parts[0])
            else:
                # Always save the Deployment when a new dive file is added
                # so a checksum is calculated and a new deployment.json file
                # is created
                fname, ext = os.path.splitext(path_parts[-1])
                if ext not in ['.md5', '.txt']:
                    deployment.save()
                    logger.info("Updated deployment %s", path_parts[0])
Ejemplo n.º 12
0
#!/usr/bin/env python
'''
scripts/update_deployments.py

A script to update (save) all the deployments in mongo (Deployment model)

It can be used to properly update all the old deployments latest_file and
latest_file_mtime attributes. Those attributes are updated in the .save
method of the Deployment model.

This is not a script that should be run on a schedule. Only one-offs as
necessary from within the container.
'''
from glider_dac import app, db

if __name__ == '__main__':
    with app.app_context():
        deployments = db.Deployment.find()
        for d in deployments:
            d.save()
Ejemplo n.º 13
0
    '''
    Gets the best candidate for a WMO ID

    :param netCDF4.Dataset nc: An open netCDF4 Dataset
    '''
    if getattr(nc, 'wmo_id', None):
        wmo_id = nc.wmo_id
    elif 'wmo_id' in nc.variables:
        wmo_id = ''.join(nc.variables['wmo_id'][:].flatten())
    else:
        return None
    return wmo_id


def get_acknowledgment(nc):
    '''
    Gets the best candidate for an acknowledgment

    :param netCDF4.Dataset nc: An open netCDF4 Dataset
    '''
    return getattr(nc, 'acknowledgment', None)

if __name__ == '__main__':
    from argparse import ArgumentParser
    parser = ArgumentParser(description=main.__doc__)

    args = parser.parse_args()
    with app.app_context():
        sys.exit(main(args))

Ejemplo n.º 14
0
def glider_deployment_check(data_type=None,
                            completed=True,
                            force=False,
                            deployment_dir=None,
                            username=None):
    """
    """
    cs = CheckSuite()
    cs.load_all_available_checkers()
    with app.app_context():
        if data_type is not None:
            is_delayed_mode = data_type == 'delayed'
            if is_delayed_mode:
                q_dict = {"delayed_mode": True, "completed": completed}
            else:
                q_dict = {
                    "$or": [{
                        "delayed_mode": False
                    }, {
                        "delayed_mode": {
                            "$exists": False
                        }
                    }],
                    "completed":
                    completed
                }

            if not force:
                q_dict["compliance_check_passed"] = {"$ne": True}

        # TODO: combine username/deployment cases?
        if username:
            q_dict = {"username": username}
        # a particular deployment has been specified
        elif deployment_dir:
            q_dict = {"deployment_dir": deployment_dir}
        else:
            q_dict = {}

        agg_pipeline = [{
            "$match": q_dict
        }, {
            "$group": {
                "_id": "$user_id",
                "deployments": {
                    "$push": {
                        "_id": "$_id",
                        "name": "$name",
                        "deployment_dir": "$deployment_dir"
                    }
                }
            }
        }]
        # if force is enabled, re-check the datasets no matter what

        # is this syntax still used?  if the first fn call fails, use the
        # second set of results
        try:
            agg_result_set = db.deployments.aggregate(agg_pipeline)['result']
        except:
            agg_result_set = db.deployments.aggregate(agg_pipeline, cursor={})
        for res in agg_result_set:
            user = db.users.find_one(res["_id"])
            all_messages = []
            failing_deployments = []
            for dep in res['deployments']:
                root_logger.info("Running compliance check on glider "
                                 "deployment: {}".format(dep))
                try:
                    dep_passed, dep_messages = process_deployment(dep)
                    all_messages.append(dep_messages)
                    if not dep_passed:
                        failing_deployments.append(dep)
                except Exception as e:
                    root_logger.exception(
                        "Exception occurred while processing deployment {}".
                        format(dep['name']))
                    text_body = ''
            send_deployment_cchecker_email(user, failing_deployments,
                                           "\n".join(all_messages))
Ejemplo n.º 15
0
def main():
    with app.app_context():
        update()
Ejemplo n.º 16
0
def main():
    with app.app_context():
        update()
Ejemplo n.º 17
0
 def wrapper(*args, **kwargs):
     with app.app_context():
         return f(*args, **kwargs)