Пример #1
0
def main():
    opts = parse_args()
    if not opts.mime_type:
		print "Exactly one mime-type must be given!"
		exit(1)
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
            client_id=cfg['googledrive']['client id'],
            client_secret=cfg['googledrive']['client secret'],
            scopes=[DRIVE_RW_SCOPE],
            )

    # Establish our credentials.
    gd.authenticate()

    # Get information about the specified file.  This will throw
    # an exception if the file does not exist.
    md = gd.get_file_metadata(opts.docid)

    # Initialize the git repository.
    print 'Create repository "%(title)s"' % md
    subprocess.call(['git','init',md['title']])
    os.chdir(md['title'])

    # Iterate over the revisions (from oldest to newest).
    for rev in gd.revisions(opts.docid):
        with open('content', 'w') as fd:
            if 'exportLinks' in rev and not opts.raw:
                # If the file provides an 'exportLinks' dictionary,
                # download the requested MIME type.
                r = gd.session.get(rev['exportLinks'][opts.mime_type])
            elif 'downloadUrl' in rev:
                # Otherwise, if there is a downloadUrl, use that.
                r = gd.session.get(rev['downloadUrl'])
            else:
                raise KeyError('unable to download revision')

            # Write file content into local file.
            for chunk in r.iter_content():
                fd.write(chunk)

        # Commit changes to repository.
        subprocess.call(['git', 'add', 'content'])
        subprocess.call(['git', 'commit', '-m',
            'revision from %s' % rev['modifiedDate']])
Пример #2
0
def main():
    opts = parse_args()
    if not opts.mime_type:
        print("Exactly one mime-type must be given!")
        exit(1)
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
        client_id=cfg['googledrive']['client id'],
        client_secret=cfg['googledrive']['client secret'],
        scopes=[DRIVE_RW_SCOPE],
    )

    # Establish our credentials.
    gd.authenticate()

    # Get information about the specified file.  This will throw
    # an exception if the file does not exist.
    md = gd.get_file_metadata(opts.docid)

    # Precalc some directory and filename constants:
    extension = filetype = MIME_TYPE_EXTENSIONS[opts.mime_type]
    gitrepo = u'%s/%s' % (filetype, md['title'])
    filename = u'content.%s' % extension

    if os.path.exists(gitrepo):
        sys.stderr.write('Error: repo "%s" already exists.\n' % gitrepo)
        sys.exit(1)

    # Initialize the git repository.
    print('Create repository %s' % gitrepo)
    subprocess.call(['git', 'init', gitrepo])
    os.chdir(gitrepo)

    # Iterate over the revisions (from oldest to newest).
    for rev in gd.revisions(opts.docid):
        with open(filename, 'w') as fd:
            if 'exportLinks' in rev and not opts.raw:
                # If the file provides an 'exportLinks' dictionary,
                # download the requested MIME type.
                r = gd.session.get(rev['exportLinks'][opts.mime_type])
            elif 'downloadUrl' in rev:
                # Otherwise, if there is a downloadUrl, use that.
                r = gd.session.get(rev['downloadUrl'])
            else:
                raise KeyError('unable to download revision')

            # Write file content into local file.
            for chunk in r.iter_content():
                fd.write(chunk)

        # Commit changes to repository.
        subprocess.call(['git', 'add', filename])
        subprocess.call([
            'git', 'commit', '-m',
            'url: %s' % rev['selfLink'],
            '--author="%s <%s>"' % (rev['lastModifyingUser']['displayName'],
                                    rev['lastModifyingUser']['emailAddress']),
            '--date="%s"' % rev['modifiedDate']
        ])
Пример #3
0
def main():
    opts = parse_args()
    if not opts.mime_type:
        print("Exactly one mime-type must be given!")
        exit(1)
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
        client_id=cfg['googledrive']['client id'],
        client_secret=cfg['googledrive']['client secret'],
        scopes=[DRIVE_RW_SCOPE],
    )

    # Establish our credentials.
    gd.authenticate()

    # Get information about the specified file.  This will throw
    # an exception if the file does not exist.
    md = gd.get_file_metadata(opts.docid)

    # Initialize the git repository.
    print('Create repository "%(title)s"' % md)
    subprocess.call(['git', 'init', md['title']])
    os.chdir(md['title'])

    # Iterate over the revisions (from oldest to newest).
    for rev in gd.revisions(opts.docid):
        with open('content', 'wb') as fd:
            if 'exportLinks' in rev and not opts.raw:
                # If the file provides an 'exportLinks' dictionary,
                # download the requested MIME type.

                if not (opts.mime_type in rev['exportLinks']):
                    print("Specified mimetype '", opts.mime_type,
                          "' does not exist for specified document.")
                    print("Available mimetypes for this document are:")
                    for key in rev['exportLinks'].keys():
                        print("- ", key)
                    exit(1)

                r = gd.session.get(rev['exportLinks'][opts.mime_type])
            elif 'downloadUrl' in rev:
                # Otherwise, if there is a downloadUrl, use that.
                r = gd.session.get(rev['downloadUrl'])
            else:
                raise KeyError('unable to download revision')

            # Write file content into local file.
            for chunk in r.iter_content():
                fd.write(chunk)

        # Commit changes to repository.
        subprocess.call(['git', 'add', 'content'])
        subprocess.call(
            ['git', 'commit', '-m',
             'revision from %s' % rev['modifiedDate']])
Пример #4
0
def main():
    opts = parse_args()
    if not opts.mime_type:
        print "Exactly one mime-type must be given!"
        exit(1)
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
        client_id=cfg['googledrive']['client id'],
        client_secret=cfg['googledrive']['client secret'],
        scopes=[DRIVE_RW_SCOPE],
    )

    # Establish our credentials.
    gd.authenticate()

    # Get information about the specified file.  This will throw
    # an exception if the file does not exist.
    md = gd.get_file_metadata(opts.docid)

    # Initialize the git repository.
    print 'Create repository "%(title)s"' % md
    subprocess.call(['git', 'init', md['title']])
    os.chdir(md['title'])

    # Iterate over the revisions (from oldest to newest).
    for rev in gd.revisions(opts.docid):
        # print rev
        with open(opts.output, 'w') as fd:
            if 'exportLinks' in rev and not opts.raw:
                # If the file provides an 'exportLinks' dictionary,
                # download the requested MIME type.
                r = gd.session.get(rev['exportLinks'][opts.mime_type])
            elif 'downloadUrl' in rev:
                # Otherwise, if there is a downloadUrl, use that.
                r = gd.session.get(rev['downloadUrl'])
            else:
                raise KeyError('unable to download revision')

            # Write file content into local file.
            for chunk in r.iter_content():
                fd.write(chunk)

        # Commit changes to repository.
        subprocess.call(['git', 'add', opts.output])
        subprocess.call([
            'git',
            'commit',
            '--author="%s <%s>"' %
            (rev.get('lastModifyingUserName'), rev['lastModifyingUser'].get(
                'emailAddress', '')),
            '-m',
            'doc: import revision from %s' % rev['modifiedDate'],
        ])
Пример #5
0
def main():
    opts = parse_args()
    if not opts.mime_types:
        print('At least one mime-type must be given: ensure that you are running with -T, -H, etc.')
        exit(1)
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
            client_id=cfg['googledrive']['client id'],
            client_secret=cfg['googledrive']['client secret'],
            scopes=[DRIVE_RW_SCOPE],
            )

    # Establish our credentials.
    gd.authenticate()

    # Get information about the specified file.  This will throw
    # an exception if the file does not exist.
    md = gd.get_file_metadata(opts.docid)

    if os.path.isdir(md['title']):
        # Find revision matching last commit and process only following revisions
        os.chdir(md['title'])
        print('Update repository "{0}"'.format(md['title'].encode('utf-8')))
        last_commit_message = subprocess.check_output('git log -n 1 --format=%B', shell=True).decode(sys.stdout.encoding)
        print('Last commit: ' + last_commit_message + 'Iterating Google Drive revisions:')
        revision_matched = False
        for rev in gd.revisions(opts.docid):
            if revision_matched:
                print('New revision: ' + rev['modifiedDate'])
                commit_revision(gd, opts, rev)
            if rev['modifiedDate'] in last_commit_message:
                print('Found matching revision: ' + rev['modifiedDate'])
                revision_matched = True
        print('Repository is up to date.')
    else:
        # Initialize the git repository.
        print('Create repository "{0}"'.format(md['title'].encode('utf-8')))
        subprocess.call(['git','init',md['title']])
        os.chdir(md['title'])

        # Iterate over the revisions (from oldest to newest).
        for rev in gd.revisions(opts.docid):
            commit_revision(gd, opts, rev)
Пример #6
0
    # Prepare output repository.
    if os.path.exists(opts.output):
        if opts.force:
            shutil.rmtree(opts.output)
        else:
            logging.error(
                "There is already a file/directory at %s, use --force to overwrite",
                opts.output)
            sys.exit(2)
    os.makedirs(opts.output)

    # Establish our credentials.
    cfg = yaml.load(open(opts.config))
    gd = GoogleDrive(
        client_id=cfg["googledrive"]["client id"],
        client_secret=cfg["googledrive"]["client secret"],
        scopes=[DRIVE_RW_SCOPE],
    )
    gd.authenticate()

    # Scan for events.
    s = EventScanner(gd, opts)
    s.scan(opts.id)
    logging.info("Scan completed, found %d events", len(s.events))
    # Initialize the git repository.
    repo = git.init_repository(opts.output)
    logging.info("Created repository at %s", repo.workdir)
    c = EventCommitter(gd, opts, repo)
    c.commit(s.events)
    logging.info("Complete!")
Пример #7
0
def main():
    opts = parse_args()
    cfg = yaml.load(open(opts.config))

    # match Yaml options to commandline options
    try:
        if cfg['mail']['frommail'] != "":
            opts.frommail = cfg['mail']['frommail']
    except:
        pass
    try:
        if cfg['mail']['tomail'] != "":
            opts.tomail = cfg['mail']['tomail']
    except:
        pass
    try:
        if cfg['mail']['smtphost'] != "":
            opts.smtphost = cfg['mail']['smtphost']
    except:
        pass
    try:
        if cfg['mail']['smtpport'] != "":
            opts.smtpport = int(cfg['mail']['smtpport'])
    except:
        pass
    try:
        if cfg['mail']['usessl'] != "":
            opts.usessl = bool(cfg['mail']['usessl'])
    except:
        pass
    try:
        if cfg['mail']['username'] != "":
            opts.username = cfg['mail']['username']
    except:
        pass
    try:
        if cfg['mail']['password'] != "":
            opts.password = cfg['mail']['password']
    except:
        pass
    try:
        if cfg['interval'] != "":
            opts.interval = int(cfg['interval'])
    except:
        pass
    try:
        if cfg['update'] != "":
            opts.update = bool(cfg['update'])
    except:
        pass
    try:
        if cfg['queries'] != "":
            opts.queries = cfg['queries']
    except:
        pass
    try:
        if cfg['slackroom'] != "":
            opts.slackroom = cfg['slackroom']
    except:
        pass
    try:
        if cfg['slacktoken'] != "":
            opts.slacktoken = cfg['slacktoken']
    except:
        pass
    try:
        if cfg['slackuser'] != "":
            opts.slackuser = cfg['slackuser']
    except:
        pass
    try:
        if cfg['verbose'] != "":
            opts.verbose = bool(cfg['verbose'])
    except:
        pass

    # print debug info
    if opts.verbose:
        print("from mail: " + opts.frommail)
        print("to mail: " + str(opts.tomail))
        print("smtp host: " + opts.smtphost)
        print("smtp port: " + str(opts.smtpport))
        print("use ssl: " + str(opts.usessl))
        print("username: "******"interval: " + str(opts.interval))
        print("update: " + str(opts.update))
        print("queries: " + str(opts.queries))
        print("slack room: " + str(opts.slackroom))
        print("slack user: "******"Adding document: " + md['title'] + " (" + file['id'] + ")")
                        docs[file['id']] = None

            # Check that we are not getting too many documents
            if counter > 999:
                print("WARNING: Query returns too many documents, not tracking all docs: " + query)

        # Track if there where changes
        foundChanges = False

        # A mail message to send via mail
        mailMessageBody = ""

        # A message to be sent via slack
        slackMessageBody = ""

        # Iterate over the documents we monitor
        for doc,modifiedDate in docs.items():
            # Get information about the specified file.  This will throw
            # an exception if the file does not exist.
            md = gd.get_file_metadata(doc)

            if md['mimeType'] == 'application/vnd.google-apps.document':
                # Get some metadata for the file we track
                if modifiedDate is None:
                    modifiedDate=md['modifiedDate']
                title = md['title']
                try:
                    md['embedLink'] 
                except KeyError:
                    editLink = ""
                else:
                    editLink = md['embedLink'].replace('preview', 'edit')

                # Iterate over the revisions (from oldest to newest).
                lastRevModifiedDate=None
                for rev in gd.revisions(doc):
                    lastRevModifiedDate = rev['modifiedDate']
                if modifiedDate != lastRevModifiedDate:
                    foundChanges = True
                    if lastRevModifiedDate is None:
                        print("Need Edit access to: " + title + " (" + editLink + ")")
                        mailMessageBody += '<li><a href="' + editLink + '">' + title + ' (need edit access)</a></li>' + os.linesep
                        slackMessageBody += '<' + editLink + '|*' + title + ' (need edit access)*>' + os.linesep
                    else:
                        print("Document Change: " + title + " (" + editLink + ")")
                        mailMessageBody += '<li><a href="' + editLink + '">' + title + '</a></li>' + os.linesep
                        slackMessageBody += '<' + editLink + '|*' + title + '*>' + os.linesep
                    if lastRevModifiedDate is None:
                        docs[md['id']] = modifiedDate
                    else:
                        docs[md['id']] = lastRevModifiedDate

        # Write out the new files list and dates
        if opts.update and foundChanges:
            print("Updating file list")

            # write new file list
            f = open('myfilelist.py.new', 'w')
            f.write("docs={}\n")

            # Iterate over the documents we monitor
            for doc,modifiedDate in docs.items():
                if modifiedDate is not None:
               	    f.write('docs["' + doc + '"] = "' + modifiedDate + '"\n')
            f.close()

            # put new file in place
            os.rename('myfilelist.py.new', 'myfilelist.py')

        if opts.slackuser != "" and opts.slackroom != "" and opts.slacktoken != "" and slackMessageBody != "":
            print("Sending slack with changes")
            
            client = SlackClient(opts.slacktoken)
            client.chat_post_message("#" + opts.slackroom, ">>> The following documents have been changed since the last scan" + os.linesep + slackMessageBody, username=opts.slackuser)
            
        # Send a mail with the changes
        #
        # If you get: smtplib.SMTPAuthenticationError: (534... when using gmail
        #   you need to enable less secure access at https://www.google.com/settings/u/1/security/lesssecureapps
        #
        if opts.frommail != "" and mailMessageBody != "":
            print("Sending email with changes")

            finalMessageBody = "<html><head></head><body><ul>" + mailMessageBody + "</ul></body>"

            msg = MIMEText(finalMessageBody, 'html')
            msg['Subject'] = "The following documents have been changed since the last scan"
            msg['From'] = opts.frommail
            msg['To'] = ', '.join(opts.tomail)

            # Send the message via our own SMTP server, but don't include the
            # envelope header.
            if opts.usessl:
                s = smtplib.SMTP_SSL(opts.smtphost, opts.smtpport)
            else:
                s = smtplib.SMTP(opts.smtphost, opts.smtpport)
            s.ehlo()
            s.starttls()
            if opts.username != "":
                s.login(opts.username, opts.password)
            s.sendmail(opts.frommail, opts.tomail, msg.as_string())
            s.quit()

        # Bail out if we don't want to loop
        if opts.interval == 0:
            break
        else:
            time.sleep(opts.interval * 60)