def test_media_inmemory_upload_json_roundtrip(self): media = MediaInMemoryUpload(os.urandom(64), "text/plain", chunksize=10, resumable=True) data = media.to_json() newmedia = MediaInMemoryUpload.new_from_json(data) self.assertEqual(media._body, newmedia._body) self.assertEqual(media._chunksize, newmedia._chunksize) self.assertEqual(media._resumable, newmedia._resumable) self.assertEqual(media._mimetype, newmedia._mimetype)
def test_media_inmemory_upload_json_roundtrip(self): media = MediaInMemoryUpload(os.urandom(64), 'text/plain', chunksize=10, resumable=True) data = media.to_json() newmedia = MediaInMemoryUpload.new_from_json(data) self.assertEqual(media._body, newmedia._body) self.assertEqual(media._chunksize, newmedia._chunksize) self.assertEqual(media._resumable, newmedia._resumable) self.assertEqual(media._mimetype, newmedia._mimetype)
def update_file_content(self, file_id, content): """Update the file's contents of the given id on google drive.""" item = self.cache.get(file_id, None) if not item or item.metadata is None or item.expired: try: metadata = self.service.files().get(fileId=file_id).execute() except errors.HttpError as e: raise OperationFailedError(opname='update_file_content', msg=e.resp.reason) except: return self._retry_operation(self.update_file_content, file_id, content) self.cache.set(metadata['id'], metadata) else: metadata = item.metadata media_body = MediaInMemoryUpload(content) try: updated_file = self.service.files().update(fileId=file_id, body=metadata, media_body=media_body ).execute() except errors.HttpError as e: raise OperationFailedError(opname='update_file_content', msg=e.resp.reason) except TypeError as e: raise ResourceInvalidError("update_file_content %r" % e) except: return self._retry_operation(self.update_file_content, file_id, content) self.cache.pop(file_id, None) self._add_to_cache_from_dict(updated_file, updated_file['parents']) return updated_file
def new_block(self, idx, data=None): with self.lock: if idx >= self.block_count or idx < 0: raise ValueError("Index out of bound") if self.mapping[idx] is not None: raise ValueError("None empty mapping @ {0}".format(idx)) if data is not None: assert len(data) == self.block_size else: data = "\0" * self.block_size body = { 'title': self.idx_to_name(idx), 'mimeType': self.BLOCK_MIMETYPE, 'parents': [{ 'id': self.data_dir }], } media_body = MediaInMemoryUpload(data, mimetype=self.BLOCK_MIMETYPE, resumable=False) result = self.drive.files().insert( body=body, media_body=media_body).execute() self.mapping[idx] = result['id'] return result
def init_data_dir(self): logger.info("Initializing data dir") if 'default_block_size' in self.config: block_size = int(self.config['default_block_size']) else: block_size = int(raw_input("Desired block size: ")) if 'default_total_size' in self.config: total_size = int(self.config['default_total_size']) else: total_size = int(raw_input("Total size: ")) if total_size < block_size: raise ValueError("block_size should not be bigger than total_size.") used_size = total_size // block_size * block_size if used_size != total_size: logger.info("Only using {0} bytes instead of {1}".format(used_size, total_size)) self.bd_attr = { 'version': Metadata['version'], 'block_size': block_size, 'block_count': used_size // block_size, } body = { 'title': 'config', 'description': 'config file for gbd', 'mimeType': 'application/json', 'parents': [{'id': self.data_dir}], } media_body = MediaInMemoryUpload(json.dumps(self.bd_attr), mimetype='application/json', resumable=False) self.drive.files().insert(body=body, media_body=media_body).execute()
def upload_file(file_data, file_name, mime='application/octet-stream'): try: open(token_file) except IOError: http = authorize(token_file, create_token_file(token_file)) http = authorize(token_file, None) drive_service = build('drive', 'v2', http=http) file_data = base64.b64decode(file_data) media_body = MediaInMemoryUpload(file_data, mimetype=mime, resumable=True) body = { 'title': file_name, 'description': 'File from OpenERP', 'mimeType': mime, } permissions = { 'role': 'reader', 'type': 'anyone', 'value': None, 'withLink': True } # Заливаем файл file = drive_service.files().insert(body=body, media_body=media_body).execute() # Устанавливаем ему права drive_service.permissions().insert(fileId=file['id'], body=permissions).execute() # Получаем инстанс файла и линк на скачку file = drive_service.files().get(fileId=file['id']).execute() download_url = file.get('webContentLink') return download_url
def shatner (request): da = DriveAuth(request) creds = da.get_session_credentials() if creds is None: return http.HttpResponseForbidden('Login Again') task = request.POST.get('task', '') if task in ('open', 'save'): service = CreateService('drive', 'v2', creds) if service is None: return http.HttpResponseServerError('Something broke') if task == 'open': file_id = request.POST.get('file_id', '') if file_id: try: f = service.files().get(fileId=file_id).execute() except AccessTokenRefreshError: return http.HttpResponseForbidden('Login Again') downloadUrl = f.get('downloadUrl') f['content'] = '' if downloadUrl: resp, f['content'] = service._http.request(downloadUrl) return http.HttpResponse( json.dumps({'status': 'ok', 'file': f}), content_type='application/json' ) elif task == 'save': mt = 'text/plain' name = request.POST.get('name') content = request.POST.get('content', '') file_id = request.POST.get('file_id', '') resource = {'title': name, 'mimeType': mt} file = MediaInMemoryUpload(content.encode('utf-8'), mt) try: google = service.files().update( fileId=file_id, newRevision=True, body=resource, media_body=file, useContentAsIndexableText=True, ).execute() except AccessTokenRefreshError: return http.HttpResponseForbidden('Login Again') return http.HttpResponse( json.dumps({'status': 'ok'}), content_type='application/json' ) return http.HttpResponseBadRequest('Bad Request')
def createSpreadsheet(self, sheetName, description=None, parentId=None, csv=None, rows=1000, cols=26): ''' sheetName: file name for Google Drive description: description for Google Drive (default same as sheetName) parentId: create into parent folder (default 'root') csv: string (default None: empty sheet) rows: int (default 1000) cols: int (default 26) ''' body = { 'title': sheetName, 'mimeType': 'text/csv', # to be converted 'description': description if description else sheetName } if parentId is None: parentId = 'root' body['parents'] = [{'id': parentId}] mbody = MediaInMemoryUpload(csv if csv else '\n'.join([',' * cols] * rows), mimetype='text/csv', chunksize=256 * 1024, resumable=False) req = self.service.files().insert(body=body, media_body=mbody) req.uri += '&convert=true' fileObj = req.execute() if fileObj is None: return (None, None) self.sheetId = fileObj['id'] self.set_activesheet() return (self.sheetId, fileObj)
def write_block(self, idx, data): assert len(data) == self.gbd.block_size blkid = self.gbd.block_id(idx) if blkid is None: return self.gbd.new_block(idx, data) else: media_body = MediaInMemoryUpload(data, mimetype=self.gbd.BLOCK_MIMETYPE, resumable=False) return self.drive.files().update(fileId=blkid, media_body=media_body).execute()
def test_media_inmemory_upload(self): media = MediaInMemoryUpload('abcdef', mimetype='text/plain', chunksize=10, resumable=True) self.assertEqual('text/plain', media.mimetype()) self.assertEqual(10, media.chunksize()) self.assertTrue(media.resumable()) self.assertEqual('bc', media.getbytes(1, 2)) self.assertEqual(6, media.size())
def upload_string(self, account_id, profile_id, string, data_source_id): """ Upload to custom data sources - example function https://developers.google.com/analytics/devguides/config/mgmt/v3/mgmtReference/management/uploads/uploadData """ analytics = self.get_service_object(name='management') media = MediaInMemoryUpload(string, mimetype='application/octet-stream', resumable=False) analytics.management().uploads().uploadData( accountId=account_id, webPropertyId=profile_id, customDataSourceId=data_source_id, media_body=media).execute()
def test_media_inmemory_upload(self): media = MediaInMemoryUpload("abcdef", "text/plain", chunksize=10, resumable=True) self.assertEqual("text/plain", media.mimetype()) self.assertEqual(10, media.chunksize()) self.assertTrue(media.resumable()) self.assertEqual("bc", media.getbytes(1, 2)) self.assertEqual(6, media.size())
def main(): logging.basicConfig(level=logging.INFO, format='%(levelname)-8s: %(message)s') parser = gdrive.get_argparser(description=__doc__.strip()) parser.add_argument('filename', help="Beancount input file") parser.add_argument('docid', help="Document ID") parser.add_argument( '-o', '--output', action='store', default=datetime.date.today().strftime('beancount.%Y-%m-%d.pdf'), help="Where to write out the collated PDF file") args = parser.parse_args() # Load the file. entries, unused_errors, options_map = loader.load_file( args.filename, log_timings=logging.info, log_errors=sys.stderr) # Generate a report. holdings_list = holdings_reports.report_holdings( None, False, entries, options_map, aggregation_key=lambda holding: holding.currency) oss = io.StringIO() table.table_to_csv(holdings_list, file=oss) csv_contents = oss.getvalue() # Connect, with authentication. # Check https://developers.google.com/drive/scopes for all available scopes. scopes = [ 'https://www.googleapis.com/auth/drive', ] http = gdrive.get_authenticated_http(" ".join(scopes), args) # Access the drive API. drive = gdrive.discovery.build('drive', 'v2', http=http) # Get the document and replace it. metadata = drive.files().get(fileId=args.docid).execute() upload = MediaInMemoryUpload(csv_contents.encode('utf-8'), mimetype=metadata['mimeType']) metadata = drive.files().update(fileId=args.docid, media_body=upload).execute() pprint(metadata)
def insert(self, rows): ''' rows は (refId, id, similarity) のリストまたはタプル ''' rowsstr = '\r\n'.join([ ','.join(row) for row in [map(lambda v: str(v), row) for row in rows] ]) media = MediaInMemoryUpload(body=rowsstr.encode('UTF-8'), mimetype='application/octet-stream', resumable=True) return self.service.table().importRows(tableId=self.table_id, media_body=media, encoding='UTF-8').execute()
def download_images(urls, tag, photo_info): #get service, then create a new folder based on each tag service = get_credentials() folder_id = create_folder(tag) # makes sure path exists #for loop through urls to create image in Drive for url in urls: image_name = url.split("/")[-1] file_metadata = {'name': image_name, 'parents': [folder_id]} #response opens up the data. then image_file stores it response = urllib.request.urlopen(url) image_file = io.BytesIO(response.read()) #create Drive media, then upload it media = MediaInMemoryUpload(image_file.read(), mimetype='image/jpeg', resumable=True) file = service.files().create(body=file_metadata, media_body=media, fields='id').execute() #create CSV based on all images' information with certain tags with open('coordinates.csv', 'w', newline='') as csvfile: writer = csv.writer(csvfile, quoting=csv.QUOTE_NONE) #write in field names, then add info from data into the fields writer.writerow([ 'Latitude', 'Longitude', "PhotoID", "Description", "Title", "Taken", "Real Name", "User Name" ]) for field in photo_info: writer.writerow([ field[0], field[1], field[2], field[3], field[4], field[5], field[6], field[7] ]) csv_file_metadata = {'name': tag, 'parents': [folder_id]} #create mdeia file for csv and upload it media = MediaFileUpload('coordinates.csv', mimetype='text/csv', resumable=True) file = service.files().create(body=csv_file_metadata, media_body=media, fields='id').execute()
def replace_gdocs_document(http, docid, title, contents): """Upload new contents for a Google Doc for a plain/text file. Args: http: An http connection object with drive credentials. docid: A string, the ID of the document. title: A string, the title of the document. contents: A string, the body of the document. """ service = discovery.build('drive', 'v3', http=http) media = MediaInMemoryUpload(contents.encode('utf8'), mimetype='text/plain', resumable=True) return service.files().update(fileId=docid, body=dict(name=title), media_body=media).execute()
def uploadFromMemory(self, buff, title=None, mimetype=None): #media_body = MediaFileUpload(file_path_or_url, mimetype='text/plain', resumable=True) #media_body = MediaIoBaseUpload(open(file_path_or_url), mimetype='image/plain', resumable=True) media_body = MediaInMemoryUpload(buff, mimetype=mimetype, resumable=True) #im = cStringIO.StringIO(page.read()) #media_body = MediaIoBaseUpload(im, mimetype=page.headers['Content-Type'], resumable=True) body = {'title': title, 'mimeType': mimetype} rv = None try: rv = self.getDriveService().files().insert( body=body, media_body=media_body).execute() except errors.HttpError, error: pass
def put(self, path, content): """Upload the file Args: path: string content: string, size <= 5MB Returns: success: boolean Raises: apiclient.errors.HttpError if the response was not a 2xx. httplib2.HttpLib2Error if a transport error has occured. """ uri = GoogleAPI.UPLOAD_URL + '/files?uploadType=multipart&alt=json' path = util.format_path(path) name = os.path.basename(path) folder = os.path.dirname(path) parent_id = None beg = time.time() if folder != '/': parent = self._path_to_metadata(folder, isfolder=True) if not parent: # if the parent folder doesn't exist, then create one self.putdir(folder) parent = self._path_to_metadata(folder, isfolder=True) parent_id = parent['id'] end = time.time() dbg.google_time("get parent: %s" % (end - beg)) beg = time.time() media_body = MediaInMemoryUpload(content) body = {'title': name, 'mimeType': 'application/octet-stream'} if parent_id: # if not at root folder body['parents'] = [{'id': parent_id}] try: drive_file = self.service.files().insert( body=body, media_body=media_body).execute() self._cache_metadata(path, drive_file) end = time.time() dbg.google_time("insert finish %s" % (end - beg)) return True except errors.HttpError, error: print 'An error occurred: %s' % error return False
def put_file(self, parent_id, title, content, description=None): """Add a file to folder of the given id.""" media_body = MediaInMemoryUpload(content) body = { 'title': title, 'description': description, 'parents': [{'id': parent_id}] } try: metadata = self.service.files().insert(body=body, media_body=media_body ).execute() except errors.HttpError as e: raise OperationFailedError(opname='put_file', msg=e.resp.reason) except TypeError as e: raise ResourceInvalidError("put_file") except: return self._retry_operation(self.put_file, parent_id, title, content, description) self.cache.set(metadata['id'], metadata, parents=[parent_id]) return metadata
def _subir_archivos(cls, service, archivos=[]): parent = cls._get_parent(service) faltantes = cls._filtrar_existentes(service, parent, archivos) logging.debug(f'faltan {len(faltantes)} archivos por subir') res = [] for datos in faltantes: path = datos['path'] archivo = datos['archivo'] meta = {'name': path, 'parents': [parent]} logging.debug(f"subiendo archivo : {path}") """ este codigo es para subirlo usando el disco como intermediario path_ = obtener_path_completo_local(datos['path']) with open(path_, 'wb') as f: contenido = datos['archivo'].contenido contenido_binario = base64.b64encode(contenido).decode('utf8') f.write(contenido_binario) media = MediaFileUpload(path_, mimetype=datos['archivo'].tipo, resumable=True) """ contenido = archivo.contenido contenido_binario = base64.b64decode(contenido.encode()) media = MediaInMemoryUpload(contenido_binario, mimetype=archivo.tipo, resumable=True) r = service.files().create(body=meta, media_body=media).execute() logging.debug(f"respuesta : {r}") res.append(r) return res
def commit(self): ''' キューのデータをDBに書き込む return: 件数 ''' print '[debug] QUEUE COUNT: %d' % (len(self.queue)) if len(self.queue) == 0: return 0 rows = '' # RssId, WordList, NumberList, NewsDate, pubDate for q in self.queue: row = u'%s,"%s","%s","%s","%s"' % (q['rss_id'], ' '.join( q['news'].noun()), ' '.join(q['news'].meta().values()), q['newsDatetime'], q['pubDate']) rows += row + '\r\n' media = MediaInMemoryUpload(body=rows.encode('UTF-8'), mimetype='application/octet-stream', resumable=True) response = self.service.table().importRows(tableId=self.table_id, media_body=media, encoding='UTF-8').execute() self.queue = [] return response[u'numRowsReceived']
def main(argv): # Declare command-line flags. # addHelp=False here because it's added downstream in the sample_init argparser = argparse.ArgumentParser(add_help=False) argparser.add_argument( '-m', '--mailbox', help='The mailbox file containing the messages to be migrated.', required=True) argparser.add_argument( '-g', '--group', help='The email address of the group to receive the migrated messages.', required=True) argparser.add_argument( '-f', '--failed', help= 'The mailbox file containing the messages that failed to be migrated.', required=False) argparser.add_argument('-v', '--verbose', help='Show progress', action='store_true') argparser.add_argument('-a', '--after', help='Only import after date', required=False) argparser.add_argument('-b', '--before', help='Only import before date', required=False) argparser.add_argument('-n', '--dryrun', help='Dry-run', action='store_true') argparser.add_argument('-r', '--resume', help='resume with message #', required=False) argparser.add_argument('-l', '--label', help='import matching label (X-Gmail-Labels)', required=False) # Authenticate and construct service scope = ("https://www.googleapis.com/auth/apps.groups.migration") service, flags = sample_tools.init(argv, 'groupsmigration', 'v1', __doc__, __file__, parents=[argparser], scope=scope) if (flags.after): flags.after = pytz.timezone('US/Pacific').localize( dateutil.parser.parse(flags.after)) logging.info("only migrating messages after date: %s" % flags.after) if (flags.before): flags.before = pytz.timezone('US/Pacific').localize( dateutil.parser.parse(flags.before)) logging.info("only migrating messages before date: %s" % flags.before) if (flags.resume): flags.resume = int(flags.resume) logging.info("resuming with id# %s" % flags.resume) if (flags.label): logging.info("only importing messags with label \"%s\"" % flags.label) mbox = mailbox.mbox(flags.mailbox, create=False) i, mboxLen = 0, len(mbox) print "mailbox size: %d messages" % mboxLen # create a new mbox with failed messages, to make reruns shorter if flags.failed: failbox = mailbox.mbox(flags.failed, create=True) failbox.lock() try: for message in mbox: i += 1 message.x_date = None message.x_labels = [] try: message.x_date = dateutil.parser.parse(message.get('Date')) try: message.x_labels = map( lambda s: s.strip(), message.get('X-Gmail-Labels').split(',')) logging.debug("labels: %s" % message.x_labels) if flags.label and len(message.x_labels) > 0 and ( flags.label not in message.x_labels): logging.debug( "skipping: \"%s\" does not contain label \"%s\"" % (message.x_labels, flags.label)) continue except: pass if (flags.after and message.x_date and (message.x_date < flags.after)): logging.debug("skipping: date %s is before flags.after" % message.x_date) continue if (flags.before and message.x_date and (message.x_date > flags.before)): logging.debug("skipping: date %s is after flags.before" % message.x_date) continue if flags.resume and (i < flags.resume): logging.debug("skipping: message_id %s is before resume" % i) continue except (ValueError, TypeError, AttributeError) as e: logging.error(str(e)) logging.debug("message-id: %s" % message['message-id']) logging.debug("subject: %s" % message['subject']) logging.debug("date: %s" % message.x_date) print "processing message %s / %s" % (i, mboxLen) if (flags.dryrun): continue media = MediaInMemoryUpload(str(message), mimetype='message/rfc822') try: request = service.archive().insert(groupId=flags.group, media_body=media) result = request.execute() logging.debug("response: %s" % result['responseCode']) if flags.failed and result['responseCode'] != 'SUCCESS': failbox.add(message) except (MediaUploadSizeError, HttpError, socket.error) as e: if flags.failed: failbox.add(message) logging.error("%s: %s" % (message['message-id'], str(e))) except AccessTokenRefreshError: logging.error( 'The credentials have been revoked or expired, please re-run the ' 'application to re-authorize') if flags.failed: failbox.flush() failbox.unlock()
def convert_raw_document(raw_document, user=None): """ Upload a raw document to google drive and get a Note back""" fp_file = raw_document.get_file() # extract some properties from the document metadata filename = raw_document.name print "this is the mimetype of the document to check:" mimetype = raw_document.mimetype print mimetype print "" # A special case for Evernotes if raw_document.mimetype == 'text/enml': raw_document.mimetype = 'text/html' original_content = fp_file.read() # Include mimetype parameter if there is one to include extra_flags = {'mimetype': raw_document.mimetype} if raw_document.mimetype \ else {} media = MediaInMemoryUpload(original_content, chunksize=1024*1024, \ resumable=True, **extra_flags) service = build_api_service() # upload to google drive file_dict = upload_to_gdrive(service, media, filename, mimetype=mimetype) # download from google drive content_dict = download_from_gdrive(service, file_dict, mimetype=mimetype) # this should have already happened, lets see why it hasn't raw_document.is_processed = True raw_document.save() note = raw_document.convert_to_note() # Cache the uploaded file's URL note.gdrive_url = file_dict['alternateLink'] note.text = content_dict['text'] # Extract HTML from the appropriate place html = '' if raw_document.mimetype == PDF_MIMETYPE: html = pdf2html(original_content) elif raw_document.mimetype in PPT_MIMETYPES: html = pdf2html(content_dict['pdf']) elif 'html' in content_dict and content_dict['html']: html = content_dict['html'] if html: html = sanitizer.data_uris_to_s3(html) NoteMarkdown.objects.create(note=note, html=html) # If we know the user who uploaded this, # associate them with the note if user and not user.is_anonymous(): note.user = user NoteKarmaEvent.create_event(user, note, NoteKarmaEvent.UPLOAD) else: try: mapping = UserUploadMapping.objects.get( fp_file=raw_document.fp_file) note.user = mapping.user note.save() NoteKarmaEvent.create_event(mapping.user, note, NoteKarmaEvent.UPLOAD) except (ObjectDoesNotExist, MultipleObjectsReturned): logger.info("Zero or multiple mappings found with fp_file " + raw_document.fp_file.name) # Finally, save whatever data we got back from google note.save()
def shatner(request): da = DriveAuth(request) creds = da.get_session_credentials() if creds is None: return JsonResponse({'status': 'auth_needed'}) task = request.POST.get('task', '') service = CreateService('drive', 'v2', creds) if service is None: return JsonResponse({'status': 'no_service'}) if task == 'delete': file_id = request.POST.get('file_id', '') fid = file_id if file_id.endswith('/'): fid = file_id[:-1] try: service.files().trash(fileId=fid).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) return JsonResponse({'status': 'ok', 'file_id': file_id}) elif task == 'revs': file_id = request.POST.get('file_id', '') try: revisions = service.revisions().list(fileId=file_id).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) else: #for item in revisions.get('items', []): # logging.info('PIN: ' + str(item['pinned'])) # logging.info('PUB: ' + str(item['published'])) # html = render_to_string('main/revs.html', { 'file_id': file_id, 'items': revisions.get('items', []) }) return JsonResponse({'status': 'ok', 'file_id': file_id, 'html': html}) elif task == 'get_url': file_id = request.POST.get('file_id', '') url = request.POST.get('url', '') try: resp, content = service._http.request(url) except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) return JsonResponse({ 'status': 'ok', 'file_id': file_id, 'text': content }) elif task == 'get_urls': file_id = request.POST.get('file_id', '') url1 = request.POST.get('url1', '') url2 = request.POST.get('url2', '') try: resp1, content1 = service._http.request(url1) resp2, content2 = service._http.request(url2) except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) return JsonResponse({ 'status': 'ok', 'file_id': file_id, 'text1': content1, 'text2': content2 }) elif task == 'open': file_id = request.POST.get('file_id', '') if file_id: try: f = service.files().get(fileId=file_id).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) downloadUrl = f.get('downloadUrl') if downloadUrl: resp, f['content'] = service._http.request(downloadUrl) else: f['content'] = '' fo = FileOpen(userid=da.userid, fileid=file_id, filename=f['title']) fo.put() return JsonResponse({'status': 'ok', 'file': f}) return JsonResponse({'status': 'Invalid File'}) elif task == 'save': name = request.POST.get('name') mimetype = request.POST.get('mimetype') content = request.POST.get('content', '') file_id = request.POST.get('file_id', '') new_file = request.POST.get('new_file') major = request.POST.get('major', '') md5hash = request.POST.get('md5hash', '') undos = int(request.POST.get('undos', '0')) new_revision = False if major == 'true': new_revision = True resource = {'title': name, 'mimeType': mimetype} encoding = chardet.detect(content)['encoding'] dump = None if encoding: try: dump = content.encode(encoding) except: dump = None if dump is None: dump = content.encode('utf-8') file = MediaInMemoryUpload(dump, mimetype) try: if new_file == 'false': google = service.files().update(fileId=file_id, newRevision=new_revision, body=resource, media_body=file).execute() else: google = service.files().insert(body=resource, media_body=file).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) else: file_id = google['id'] return JsonResponse(ok={ 'file_id': file_id, 'md5hash': md5hash, 'undos': undos }) elif task == 'rename': name = request.POST.get('name') file_id = request.POST.get('file_id', '') rtype = request.POST.get('rtype', '') resource = {'title': name} if rtype == 'd': file_id = file_id[:-1] resource['mimeType'] = 'application/vnd.google-apps.folder' else: newm, enc = mimetypes.guess_type(name) if newm: resource['mimeType'] = newm google = service.files().update(fileId=file_id, newRevision=True, body=resource).execute() parents = [] for p in google['parents']: if p['isRoot']: parents.append('') else: parents.append(p['id']) if not parents: parents.append('') return JsonResponse(ok={ 'file_id': file_id, 'parents': parents, 'name': name }) elif task == 'new': name = request.POST.get('name') parent = request.POST.get('parent', '') mime_type, enc = mimetypes.guess_type(name) root, ext = os.path.splitext(name) if not mime_type: mime_type = 'application/octet-stream' if ext: ext = ext[1:] media_body = MediaInMemoryUpload('', mime_type) body = {'title': name, 'mimeType': mime_type} if parent: body['parents'] = [{'id': parent}] try: google = service.files().insert(body=body, media_body=media_body).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) else: return JsonResponse( ok={ 'file_id': google['id'], 'title': name, 'ext': ext, 'mime': mime_type, 'url': google['alternateLink'], 'parent': parent }) elif task == 'new_dir': name = request.POST.get('name') parent = request.POST.get('parent', '') dmt = 'application/vnd.google-apps.folder' body = {'title': name, 'mimeType': dmt} if parent: body['parents'] = [{'id': parent}] try: google = service.files().insert(body=body).execute() except AccessTokenRefreshError: return JsonResponse({'status': 'auth_needed'}) else: return JsonResponse( ok={ 'file_id': google['id'], 'title': name, 'mime': dmt, 'parent': parent }) return http.HttpResponseBadRequest('Invalid Task', mimetype='text/plain')
def main(argv): """Migrates email messages from Mailman mbox archives to Google Group using the Groups Migration API.""" parser = ArgumentParser(parents=[tools.argparser]) parser.add_argument('--group', help='Group email address', required=True) parser.add_argument('--mbox', help='Mailman archive file (.mbox)', required=True) parser.add_argument('--resume', type=int, default=1, help='Resume from the specified offset') args = parser.parse_args() # Setup logging logging.basicConfig() logger = logging.getLogger("migrate") logger.setLevel(getattr(logging, args.logging_level)) settings = {} # Set up a Flow object to be used if we need to authenticate. FLOW = flow_from_clientsecrets(CLIENT_SECRETS, scope='https://www.googleapis.com/auth/apps.groups.migration', message=MISSING_CLIENT_SECRETS_MESSAGE) storage = Storage(CREDENTIAL_FILE) credentials = storage.get() if credentials is None or credentials.invalid: print 'invalid credentials' # Save the credentials in storage to be used in subsequent runs. credentials = tools.run_flow(FLOW, storage, args) # Create an httplib2.Http object to handle our HTTP requests and authorize it # with our good Credentials. http = httplib2.Http() http = credentials.authorize(http) # Load the archive file logger.info('Importing mbox file %s...', args.mbox) messages = mbox(args.mbox) logger.info('%s contains %d messages.', args.mbox, len(messages)) service = build('groupsmigration', 'v1', http=http) archive = service.archive() # Upload messages remaining_messages = islice(iter(messages), args.resume-1, None) # Only process messages after the resume point (if any) with click.progressbar(messages, label='Migrating %s' % os.path.basename(args.mbox), fill_char=click.style('#', fg='green'), show_pos=True, item_show_func=show_subject) as msgs: pos = 1 for msg in msgs: if pos < args.resume: continue try: media = MediaInMemoryUpload(msg.as_string(), mimetype='message/rfc822') result = qps_limit(archive.insert(groupId=args.group, media_body=media).execute()) if result['responseCode'] != 'SUCCESS': logger.error('Message %d failed!', pos) logger.error('Subject: "%s"', msg['subject']) logger.error('Response: "%s"', result) except Exception, e: logger.error('Message %d failed!', pos) logger.error('Subject: "%s"', msg['subject']) logger.error('Response: %s', e) pos = pos + 1