def setUp(self): self.account_info = InMemoryAccountInfo() self.cache = DummyCache() self.raw_api = RawSimulator() self.api = B2Api(self.account_info, self.cache, self.raw_api) (self.application_key_id, self.master_key) = self.raw_api.create_account()
def __init__(self, parsed_url): u""" Authorize to B2 api and set up needed variables """ duplicity.backend.Backend.__init__(self, parsed_url) global DownloadDestLocalFile, FileVersionInfoFactory try: # try to import the new b2sdk if available from b2sdk.api import B2Api from b2sdk.account_info import InMemoryAccountInfo from b2sdk.download_dest import DownloadDestLocalFile from b2sdk.exception import NonExistentBucket from b2sdk.file_version import FileVersionInfoFactory except ImportError as e: if u'b2sdk' in getattr(e, u'name', u'b2sdk'): raise try: # fall back to import the old b2 client from b2.api import B2Api from b2.account_info import InMemoryAccountInfo from b2.download_dest import DownloadDestLocalFile from b2.exception import NonExistentBucket from b2.file_version import FileVersionInfoFactory except ImportError: if u'b2' in getattr(e, u'name', u'b2'): raise raise BackendException( u'B2 backend requires B2 Python SDK (pip install b2sdk)') self.service = B2Api(InMemoryAccountInfo()) self.parsed_url.hostname = u'B2' account_id = parsed_url.username account_key = self.get_password() self.url_parts = [ x for x in parsed_url.path.replace(u"@", u"/").split(u'/') if x != u'' ] if self.url_parts: self.username = self.url_parts.pop(0) bucket_name = self.url_parts.pop(0) else: raise BackendException(u"B2 requires a bucket name") self.path = u"".join([url_part + u"/" for url_part in self.url_parts]) self.service.authorize_account(u'production', account_id, account_key) log.Log( u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" % (self.path, bucket_name, self.service.account_info.get_minimum_part_size()), log.INFO) try: self.bucket = self.service.get_bucket_by_name(bucket_name) log.Log(u"Bucket found", log.INFO) except NonExistentBucket: try: log.Log(u"Bucket not found, creating one", log.INFO) self.bucket = self.service.create_bucket( bucket_name, u'allPrivate') except: raise FatalBackendException(u"Bucket cannot be created")
def init_b2(app): global b2api from b2sdk.account_info.in_memory import InMemoryAccountInfo from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo from b2sdk.api import B2Api info = InMemoryAccountInfo() b2api = B2Api(info) b2api.authorize_account('production', app.config['B2_KEY_ID'], app.config['B2_APP_KEY'])
def __init__(self, application_key_id, application_key, bucket_name, logger): self.logger = logger info = InMemoryAccountInfo() self.b2_api = B2Api(info) self.b2_api.authorize_account("production", application_key_id, application_key) self.bucket = self.b2_api.get_bucket_by_name(bucket_name)
def setUp(self): self.bucket_name = 'my-bucket' self.simulator = RawSimulator() self.account_info = StubAccountInfo() self.api = B2Api(self.account_info, raw_api=self.simulator) (self.account_id, self.master_key) = self.simulator.create_account() self.api.authorize_account('production', self.account_id, self.master_key) self.api_url = self.account_info.get_api_url() self.account_auth_token = self.account_info.get_account_auth_token() self.bucket = self.api.create_bucket('my-bucket', 'allPublic') self.bucket_id = self.bucket.id_
def __init__(self, config): self.__log = logging.getLogger(__name__) self.__info = InMemoryAccountInfo() self.__b2_api = b2_api = B2Api(self.__info) self.__b2 = b2_api.authorize_account( "production", application_key_id=config['keyId'], application_key=config['appKey']) self.__bucket: Bucket = self.__b2_api.get_bucket_by_name( config['bucket']) self.__password: str = config.get('password') self.__pre = config.backups_path if not config.backups_path.endswith( '/') else config.backups_path[:-1] self.__pre = self.__pre.lstrip('/') if self.__pre is not None else ''
def __init__(self, date: datetime.date, upload: bool): """ :param date: Date for which documents will be fetched :param upload: Boolean for whether to upload to B2 """ self.date_str = date.isoformat() self.date = date log_filename = f"diavgeia.{datetime.datetime.now().isoformat()}.log" self.logger = get_logger(f"DiavgeiaDailyFetch.{self.date_str}", log_filename) self.async_workers = ASYNC_WORKERS self.export_dir = EXPORT_DIR self.decision_queue = None self.upload_queue = None self.upload_path = Path(B2_UPLOAD_PATH) self.crawl_queue = None self.session = None self.upload = upload if self.upload: b2_api = B2Api(InMemoryAccountInfo()) b2_api.authorize_account("production", B2_KEY_ID, B2_KEY) self.bucket = b2_api.get_bucket_by_name(BUCKET_NAME)
def __init__(self, base=None, storage=None): super(B2StorageSystem, self).__init__(location=base, base_url=None) self.base = base self.storage = storage match = re.match(r'^(b2:)?(//)?(\w+)/(.+)/?$', base) if not match: return bucket = match.group(3) self.path = match.group(4) file_name = os.path.expanduser('~/.b2_account_info-%d' % storage.id) self.api = B2Api(SqliteAccountInfo(file_name=file_name)) self.api.authorize_account( 'production', storage.credential_id, storage.credential_key, ) self.bucket = self.api.get_bucket_by_name(bucket)
def run(): """Generator that does all the work and yields the response body lines. TODO: figure out how to catch and log stack traces when this function raises an exception. Currently the log only gets the exception message. Wrapping the call at the bottom in try/except doesn't work since it's a generator. :/ """ yield HTML_HEADER.encode() yield ('<div id="progress">\nFetching %s ...<br />' % url).encode() # function to print out status while downloading def download_progress_hook(progress): status = progress.get('status') if status == 'finished': msg = '<br />Extracting audio (this can take a while)...\n' elif status == 'error': # we always get an 'error' progress when the video finishes downloading. # not sure why. ignore it. return elif status == 'downloading': p = lambda field: progress.get(field) or '' try: percent = float(p('_percent_str').strip('%') or '0') except ValueError: percent = 0 msg = ('<span><progress max="100" value="%s"></progress><br /> ' '%s of %s at %s in %s...</span>\n' % ( percent, p('_downloaded_bytes_str'), p('_total_bytes_str') or p('_total_bytes_estimate_str'), p('_speed_str'), p('_eta_str'))) else: msg = status + '<br />\n' write(msg) # fetch video info (resolves URL) to see if we've already downloaded it options = { 'outtmpl': u'/tmp/%(webpage_url)s', 'restrictfilenames': True, # don't allow & or spaces in file names 'updatetime': False, # don't set output file mtime to video mtime 'logger': logging, 'logtostderr': True, 'format': 'bestaudio/best', 'noplaylist': True, 'postprocessors': [{ 'key': 'FFmpegExtractAudio', 'preferredcodec': 'mp3', 'preferredquality': '192', }], 'progress_hooks': [download_progress_hook], } ydl = youtube_dl.YoutubeDL(options) with handle_errors(write): info = ydl.extract_info(url, download=False) # prepare_filename() returns the video filename, not the postprocessed one, # so change the extension manually. the resulting filename will look like: # '/tmp/https_-_www.youtube.com_watchv=6dyWlM4ej3Q.mp3' # # ext4 max filename length is 255 bytes, and huffduffer also silently # truncates URLs to 255 chars total, so truncate before that if necessary. filename_prefix = ydl.prepare_filename(info)[:245 - len(B2_BASE)] options['outtmpl'] = filename_prefix.replace('%', '%%') + '.%(ext)s' filename = filename_prefix + '.mp3' b2api = B2Api(InMemoryAccountInfo()) b2api.authorize_account('production', B2_KEY_ID, B2_APP_KEY) bucket = b2api.get_bucket_by_name(B2_BUCKET) # strip the filename's path, scheme, and leading www., mobile, or m. # the resulting filename will look like 'youtube.com_watchv=6dyWlM4ej3Q.mp3' b2_filename = re.sub('^https?_-_((www|m|mobile|player).)?', '', os.path.basename(filename)) b2_url = bucket.get_download_url(b2_filename) uploaded_time = datetime.datetime.now() existing = requests.head(b2_url) if existing.ok: yield 'Already downloaded! <br />\n'.encode() try: uploaded_time = datetime.datetime.utcfromtimestamp( int(existing.headers.get('X-Bz-Upload-Timestamp')) / 1000) # ms except: # missing or bad header pass else: # download video and extract mp3 yield 'Downloading (this can take a while)...<br />\n'.encode() with handle_errors(write): youtube_dl.YoutubeDL(options).download([url]) # upload to B2 yield 'Uploading to B2...<br />\n'.encode() class WriteProgress(AbstractProgressListener): def set_total_bytes(self, total): self.total = total def bytes_completed(self, sent): write('<span><progress max="100" value="%s"></progress><br /> ' '%.2fMB of %.2fMB</span>\n' % ( (sent * 100 / self.total), float(sent) / 1000000, float(self.total) / 1000000)) def close(self): pass with WriteProgress() as listener: bucket.upload_local_file(filename, b2_filename, progress_listener=listener) os.remove(filename) # generate description description = info.get('description') or '' footer = """\ Original video: %s Downloaded by http://huffduff-video.snarfed.org/ on %s Available for 30 days after download""" % ( url, uploaded_time.replace(microsecond=0).ctime()) if description: footer = """ === """ + footer max_len = 1500 - len(footer) if len(description) > max_len: description = description[:max_len] + '...' description += footer # open 'Huffduff it' page yield ("""\n<br />Opening Huffduffer dialog... <script type="text/javascript"> window.location = "https://huffduffer.com/add?popup=true&%s"; </script> """ % urllib.parse.urlencode([(k, v.encode()) for k, v in (('bookmark[url]', (b2_url)), ('bookmark[title]', info.get('title') or ''), ('bookmark[description]', description), ('bookmark[tags]', ','.join(info.get('categories') or [])), )])).encode() yield HTML_FOOTER.encode()
def __init__(self, arg): """Initialize B2 connection and credentials.""" info = SqliteAccountInfo() # creds and tokens in ~/.b2_account_info b2_api = B2Api(info) self.arg = arg