def main():
opts = parse_args()
if not opts.mime_type:
print "Exactly one mime-type must be given!"
exit(1)
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg['googledrive']['client id'],
client_secret=cfg['googledrive']['client secret'],
scopes=[DRIVE_RW_SCOPE],
)
# Establish our credentials.
gd.authenticate()
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(opts.docid)
# Initialize the git repository.
print 'Create repository "%(title)s"' % md
subprocess.call(['git','init',md['title']])
os.chdir(md['title'])
# Iterate over the revisions (from oldest to newest).
for rev in gd.revisions(opts.docid):
with open('content', 'w') as fd:
if 'exportLinks' in rev and not opts.raw:
# If the file provides an 'exportLinks' dictionary,
# download the requested MIME type.
r = gd.session.get(rev['exportLinks'][opts.mime_type])
elif 'downloadUrl' in rev:
# Otherwise, if there is a downloadUrl, use that.
r = gd.session.get(rev['downloadUrl'])
else:
raise KeyError('unable to download revision')
# Write file content into local file.
for chunk in r.iter_content():
fd.write(chunk)
# Commit changes to repository.
subprocess.call(['git', 'add', 'content'])
subprocess.call(['git', 'commit', '-m',
'revision from %s' % rev['modifiedDate']])
def main():
opts = parse_args()
if not opts.mime_type:
print("Exactly one mime-type must be given!")
exit(1)
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg['googledrive']['client id'],
client_secret=cfg['googledrive']['client secret'],
scopes=[DRIVE_RW_SCOPE],
)
# Establish our credentials.
gd.authenticate()
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(opts.docid)
# Precalc some directory and filename constants:
extension = filetype = MIME_TYPE_EXTENSIONS[opts.mime_type]
gitrepo = u'%s/%s' % (filetype, md['title'])
filename = u'content.%s' % extension
if os.path.exists(gitrepo):
sys.stderr.write('Error: repo "%s" already exists.\n' % gitrepo)
sys.exit(1)
# Initialize the git repository.
print('Create repository %s' % gitrepo)
subprocess.call(['git', 'init', gitrepo])
os.chdir(gitrepo)
# Iterate over the revisions (from oldest to newest).
for rev in gd.revisions(opts.docid):
with open(filename, 'w') as fd:
if 'exportLinks' in rev and not opts.raw:
# If the file provides an 'exportLinks' dictionary,
# download the requested MIME type.
r = gd.session.get(rev['exportLinks'][opts.mime_type])
elif 'downloadUrl' in rev:
# Otherwise, if there is a downloadUrl, use that.
r = gd.session.get(rev['downloadUrl'])
else:
raise KeyError('unable to download revision')
# Write file content into local file.
for chunk in r.iter_content():
fd.write(chunk)
# Commit changes to repository.
subprocess.call(['git', 'add', filename])
subprocess.call([
'git', 'commit', '-m',
'url: %s' % rev['selfLink'],
'--author="%s <%s>"' % (rev['lastModifyingUser']['displayName'],
rev['lastModifyingUser']['emailAddress']),
'--date="%s"' % rev['modifiedDate']
])
def main():
opts = parse_args()
if not opts.mime_type:
print("Exactly one mime-type must be given!")
exit(1)
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg['googledrive']['client id'],
client_secret=cfg['googledrive']['client secret'],
scopes=[DRIVE_RW_SCOPE],
)
# Establish our credentials.
gd.authenticate()
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(opts.docid)
# Initialize the git repository.
print('Create repository "%(title)s"' % md)
subprocess.call(['git', 'init', md['title']])
os.chdir(md['title'])
# Iterate over the revisions (from oldest to newest).
for rev in gd.revisions(opts.docid):
with open('content', 'wb') as fd:
if 'exportLinks' in rev and not opts.raw:
# If the file provides an 'exportLinks' dictionary,
# download the requested MIME type.
if not (opts.mime_type in rev['exportLinks']):
print("Specified mimetype '", opts.mime_type,
"' does not exist for specified document.")
print("Available mimetypes for this document are:")
for key in rev['exportLinks'].keys():
print("- ", key)
exit(1)
r = gd.session.get(rev['exportLinks'][opts.mime_type])
elif 'downloadUrl' in rev:
# Otherwise, if there is a downloadUrl, use that.
r = gd.session.get(rev['downloadUrl'])
else:
raise KeyError('unable to download revision')
# Write file content into local file.
for chunk in r.iter_content():
fd.write(chunk)
# Commit changes to repository.
subprocess.call(['git', 'add', 'content'])
subprocess.call(
['git', 'commit', '-m',
'revision from %s' % rev['modifiedDate']])
def main():
opts = parse_args()
if not opts.mime_type:
print "Exactly one mime-type must be given!"
exit(1)
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg['googledrive']['client id'],
client_secret=cfg['googledrive']['client secret'],
scopes=[DRIVE_RW_SCOPE],
)
# Establish our credentials.
gd.authenticate()
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(opts.docid)
# Initialize the git repository.
print 'Create repository "%(title)s"' % md
subprocess.call(['git', 'init', md['title']])
os.chdir(md['title'])
# Iterate over the revisions (from oldest to newest).
for rev in gd.revisions(opts.docid):
# print rev
with open(opts.output, 'w') as fd:
if 'exportLinks' in rev and not opts.raw:
# If the file provides an 'exportLinks' dictionary,
# download the requested MIME type.
r = gd.session.get(rev['exportLinks'][opts.mime_type])
elif 'downloadUrl' in rev:
# Otherwise, if there is a downloadUrl, use that.
r = gd.session.get(rev['downloadUrl'])
else:
raise KeyError('unable to download revision')
# Write file content into local file.
for chunk in r.iter_content():
fd.write(chunk)
# Commit changes to repository.
subprocess.call(['git', 'add', opts.output])
subprocess.call([
'git',
'commit',
'--author="%s <%s>"' %
(rev.get('lastModifyingUserName'), rev['lastModifyingUser'].get(
'emailAddress', '')),
'-m',
'doc: import revision from %s' % rev['modifiedDate'],
])
def main():
opts = parse_args()
if not opts.mime_types:
print('At least one mime-type must be given: ensure that you are running with -T, -H, etc.')
exit(1)
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg['googledrive']['client id'],
client_secret=cfg['googledrive']['client secret'],
scopes=[DRIVE_RW_SCOPE],
)
# Establish our credentials.
gd.authenticate()
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(opts.docid)
if os.path.isdir(md['title']):
# Find revision matching last commit and process only following revisions
os.chdir(md['title'])
print('Update repository "{0}"'.format(md['title'].encode('utf-8')))
last_commit_message = subprocess.check_output('git log -n 1 --format=%B', shell=True).decode(sys.stdout.encoding)
print('Last commit: ' + last_commit_message + 'Iterating Google Drive revisions:')
revision_matched = False
for rev in gd.revisions(opts.docid):
if revision_matched:
print('New revision: ' + rev['modifiedDate'])
commit_revision(gd, opts, rev)
if rev['modifiedDate'] in last_commit_message:
print('Found matching revision: ' + rev['modifiedDate'])
revision_matched = True
print('Repository is up to date.')
else:
# Initialize the git repository.
print('Create repository "{0}"'.format(md['title'].encode('utf-8')))
subprocess.call(['git','init',md['title']])
os.chdir(md['title'])
# Iterate over the revisions (from oldest to newest).
for rev in gd.revisions(opts.docid):
commit_revision(gd, opts, rev)
# Prepare output repository.
if os.path.exists(opts.output):
if opts.force:
shutil.rmtree(opts.output)
else:
logging.error(
"There is already a file/directory at %s, use --force to overwrite",
opts.output)
sys.exit(2)
os.makedirs(opts.output)
# Establish our credentials.
cfg = yaml.load(open(opts.config))
gd = GoogleDrive(
client_id=cfg["googledrive"]["client id"],
client_secret=cfg["googledrive"]["client secret"],
scopes=[DRIVE_RW_SCOPE],
)
gd.authenticate()
# Scan for events.
s = EventScanner(gd, opts)
s.scan(opts.id)
logging.info("Scan completed, found %d events", len(s.events))
# Initialize the git repository.
repo = git.init_repository(opts.output)
logging.info("Created repository at %s", repo.workdir)
c = EventCommitter(gd, opts, repo)
c.commit(s.events)
logging.info("Complete!")
def main():
opts = parse_args()
cfg = yaml.load(open(opts.config))
# match Yaml options to commandline options
try:
if cfg['mail']['frommail'] != "":
opts.frommail = cfg['mail']['frommail']
except:
pass
try:
if cfg['mail']['tomail'] != "":
opts.tomail = cfg['mail']['tomail']
except:
pass
try:
if cfg['mail']['smtphost'] != "":
opts.smtphost = cfg['mail']['smtphost']
except:
pass
try:
if cfg['mail']['smtpport'] != "":
opts.smtpport = int(cfg['mail']['smtpport'])
except:
pass
try:
if cfg['mail']['usessl'] != "":
opts.usessl = bool(cfg['mail']['usessl'])
except:
pass
try:
if cfg['mail']['username'] != "":
opts.username = cfg['mail']['username']
except:
pass
try:
if cfg['mail']['password'] != "":
opts.password = cfg['mail']['password']
except:
pass
try:
if cfg['interval'] != "":
opts.interval = int(cfg['interval'])
except:
pass
try:
if cfg['update'] != "":
opts.update = bool(cfg['update'])
except:
pass
try:
if cfg['queries'] != "":
opts.queries = cfg['queries']
except:
pass
try:
if cfg['slackroom'] != "":
opts.slackroom = cfg['slackroom']
except:
pass
try:
if cfg['slacktoken'] != "":
opts.slacktoken = cfg['slacktoken']
except:
pass
try:
if cfg['slackuser'] != "":
opts.slackuser = cfg['slackuser']
except:
pass
try:
if cfg['verbose'] != "":
opts.verbose = bool(cfg['verbose'])
except:
pass
# print debug info
if opts.verbose:
print("from mail: " + opts.frommail)
print("to mail: " + str(opts.tomail))
print("smtp host: " + opts.smtphost)
print("smtp port: " + str(opts.smtpport))
print("use ssl: " + str(opts.usessl))
print("username: "******"interval: " + str(opts.interval))
print("update: " + str(opts.update))
print("queries: " + str(opts.queries))
print("slack room: " + str(opts.slackroom))
print("slack user: "******"Adding document: " + md['title'] + " (" + file['id'] + ")")
docs[file['id']] = None
# Check that we are not getting too many documents
if counter > 999:
print("WARNING: Query returns too many documents, not tracking all docs: " + query)
# Track if there where changes
foundChanges = False
# A mail message to send via mail
mailMessageBody = ""
# A message to be sent via slack
slackMessageBody = ""
# Iterate over the documents we monitor
for doc,modifiedDate in docs.items():
# Get information about the specified file. This will throw
# an exception if the file does not exist.
md = gd.get_file_metadata(doc)
if md['mimeType'] == 'application/vnd.google-apps.document':
# Get some metadata for the file we track
if modifiedDate is None:
modifiedDate=md['modifiedDate']
title = md['title']
try:
md['embedLink']
except KeyError:
editLink = ""
else:
editLink = md['embedLink'].replace('preview', 'edit')
# Iterate over the revisions (from oldest to newest).
lastRevModifiedDate=None
for rev in gd.revisions(doc):
lastRevModifiedDate = rev['modifiedDate']
if modifiedDate != lastRevModifiedDate:
foundChanges = True
if lastRevModifiedDate is None:
print("Need Edit access to: " + title + " (" + editLink + ")")
mailMessageBody += '<li><a href="' + editLink + '">' + title + ' (need edit access)</a></li>' + os.linesep
slackMessageBody += '<' + editLink + '|*' + title + ' (need edit access)*>' + os.linesep
else:
print("Document Change: " + title + " (" + editLink + ")")
mailMessageBody += '<li><a href="' + editLink + '">' + title + '</a></li>' + os.linesep
slackMessageBody += '<' + editLink + '|*' + title + '*>' + os.linesep
if lastRevModifiedDate is None:
docs[md['id']] = modifiedDate
else:
docs[md['id']] = lastRevModifiedDate
# Write out the new files list and dates
if opts.update and foundChanges:
print("Updating file list")
# write new file list
f = open('myfilelist.py.new', 'w')
f.write("docs={}\n")
# Iterate over the documents we monitor
for doc,modifiedDate in docs.items():
if modifiedDate is not None:
f.write('docs["' + doc + '"] = "' + modifiedDate + '"\n')
f.close()
# put new file in place
os.rename('myfilelist.py.new', 'myfilelist.py')
if opts.slackuser != "" and opts.slackroom != "" and opts.slacktoken != "" and slackMessageBody != "":
print("Sending slack with changes")
client = SlackClient(opts.slacktoken)
client.chat_post_message("#" + opts.slackroom, ">>> The following documents have been changed since the last scan" + os.linesep + slackMessageBody, username=opts.slackuser)
# Send a mail with the changes
#
# If you get: smtplib.SMTPAuthenticationError: (534... when using gmail
# you need to enable less secure access at https://www.google.com/settings/u/1/security/lesssecureapps
#
if opts.frommail != "" and mailMessageBody != "":
print("Sending email with changes")
finalMessageBody = "<html><head></head><body><ul>" + mailMessageBody + "</ul></body>"
msg = MIMEText(finalMessageBody, 'html')
msg['Subject'] = "The following documents have been changed since the last scan"
msg['From'] = opts.frommail
msg['To'] = ', '.join(opts.tomail)
# Send the message via our own SMTP server, but don't include the
# envelope header.
if opts.usessl:
s = smtplib.SMTP_SSL(opts.smtphost, opts.smtpport)
else:
s = smtplib.SMTP(opts.smtphost, opts.smtpport)
s.ehlo()
s.starttls()
if opts.username != "":
s.login(opts.username, opts.password)
s.sendmail(opts.frommail, opts.tomail, msg.as_string())
s.quit()
# Bail out if we don't want to loop
if opts.interval == 0:
break
else:
time.sleep(opts.interval * 60)