def setUp(self):
self.account_info = InMemoryAccountInfo()
self.cache = DummyCache()
self.raw_api = RawSimulator()
self.api = B2Api(self.account_info, self.cache, self.raw_api)
(self.application_key_id,
self.master_key) = self.raw_api.create_account()
def __init__(self, parsed_url):
u"""
Authorize to B2 api and set up needed variables
"""
duplicity.backend.Backend.__init__(self, parsed_url)
global DownloadDestLocalFile, FileVersionInfoFactory
try: # try to import the new b2sdk if available
from b2sdk.api import B2Api
from b2sdk.account_info import InMemoryAccountInfo
from b2sdk.download_dest import DownloadDestLocalFile
from b2sdk.exception import NonExistentBucket
from b2sdk.file_version import FileVersionInfoFactory
except ImportError as e:
if u'b2sdk' in getattr(e, u'name', u'b2sdk'):
raise
try: # fall back to import the old b2 client
from b2.api import B2Api
from b2.account_info import InMemoryAccountInfo
from b2.download_dest import DownloadDestLocalFile
from b2.exception import NonExistentBucket
from b2.file_version import FileVersionInfoFactory
except ImportError:
if u'b2' in getattr(e, u'name', u'b2'):
raise
raise BackendException(
u'B2 backend requires B2 Python SDK (pip install b2sdk)')
self.service = B2Api(InMemoryAccountInfo())
self.parsed_url.hostname = u'B2'
account_id = parsed_url.username
account_key = self.get_password()
self.url_parts = [
x for x in parsed_url.path.replace(u"@", u"/").split(u'/')
if x != u''
]
if self.url_parts:
self.username = self.url_parts.pop(0)
bucket_name = self.url_parts.pop(0)
else:
raise BackendException(u"B2 requires a bucket name")
self.path = u"".join([url_part + u"/" for url_part in self.url_parts])
self.service.authorize_account(u'production', account_id, account_key)
log.Log(
u"B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" %
(self.path, bucket_name,
self.service.account_info.get_minimum_part_size()), log.INFO)
try:
self.bucket = self.service.get_bucket_by_name(bucket_name)
log.Log(u"Bucket found", log.INFO)
except NonExistentBucket:
try:
log.Log(u"Bucket not found, creating one", log.INFO)
self.bucket = self.service.create_bucket(
bucket_name, u'allPrivate')
except:
raise FatalBackendException(u"Bucket cannot be created")
def init_b2(app):
global b2api
from b2sdk.account_info.in_memory import InMemoryAccountInfo
from b2sdk.account_info.sqlite_account_info import SqliteAccountInfo
from b2sdk.api import B2Api
info = InMemoryAccountInfo()
b2api = B2Api(info)
b2api.authorize_account('production', app.config['B2_KEY_ID'],
app.config['B2_APP_KEY'])
def __init__(self, application_key_id, application_key, bucket_name,
logger):
self.logger = logger
info = InMemoryAccountInfo()
self.b2_api = B2Api(info)
self.b2_api.authorize_account("production", application_key_id,
application_key)
self.bucket = self.b2_api.get_bucket_by_name(bucket_name)
def setUp(self):
self.bucket_name = 'my-bucket'
self.simulator = RawSimulator()
self.account_info = StubAccountInfo()
self.api = B2Api(self.account_info, raw_api=self.simulator)
(self.account_id, self.master_key) = self.simulator.create_account()
self.api.authorize_account('production', self.account_id,
self.master_key)
self.api_url = self.account_info.get_api_url()
self.account_auth_token = self.account_info.get_account_auth_token()
self.bucket = self.api.create_bucket('my-bucket', 'allPublic')
self.bucket_id = self.bucket.id_
def __init__(self, config):
self.__log = logging.getLogger(__name__)
self.__info = InMemoryAccountInfo()
self.__b2_api = b2_api = B2Api(self.__info)
self.__b2 = b2_api.authorize_account(
"production",
application_key_id=config['keyId'],
application_key=config['appKey'])
self.__bucket: Bucket = self.__b2_api.get_bucket_by_name(
config['bucket'])
self.__password: str = config.get('password')
self.__pre = config.backups_path if not config.backups_path.endswith(
'/') else config.backups_path[:-1]
self.__pre = self.__pre.lstrip('/') if self.__pre is not None else ''
def __init__(self, date: datetime.date, upload: bool):
"""
:param date: Date for which documents will be fetched
:param upload: Boolean for whether to upload to B2
"""
self.date_str = date.isoformat()
self.date = date
log_filename = f"diavgeia.{datetime.datetime.now().isoformat()}.log"
self.logger = get_logger(f"DiavgeiaDailyFetch.{self.date_str}", log_filename)
self.async_workers = ASYNC_WORKERS
self.export_dir = EXPORT_DIR
self.decision_queue = None
self.upload_queue = None
self.upload_path = Path(B2_UPLOAD_PATH)
self.crawl_queue = None
self.session = None
self.upload = upload
if self.upload:
b2_api = B2Api(InMemoryAccountInfo())
b2_api.authorize_account("production", B2_KEY_ID, B2_KEY)
self.bucket = b2_api.get_bucket_by_name(BUCKET_NAME)
def __init__(self, base=None, storage=None):
super(B2StorageSystem, self).__init__(location=base, base_url=None)
self.base = base
self.storage = storage
match = re.match(r'^(b2:)?(//)?(\w+)/(.+)/?$', base)
if not match:
return
bucket = match.group(3)
self.path = match.group(4)
file_name = os.path.expanduser('~/.b2_account_info-%d' % storage.id)
self.api = B2Api(SqliteAccountInfo(file_name=file_name))
self.api.authorize_account(
'production',
storage.credential_id,
storage.credential_key,
)
self.bucket = self.api.get_bucket_by_name(bucket)
def run():
"""Generator that does all the work and yields the response body lines.
TODO: figure out how to catch and log stack traces when this function raises
an exception. Currently the log only gets the exception message. Wrapping
the call at the bottom in try/except doesn't work since it's a generator. :/
"""
yield HTML_HEADER.encode()
yield ('<div id="progress">\nFetching %s ...<br />' % url).encode()
# function to print out status while downloading
def download_progress_hook(progress):
status = progress.get('status')
if status == 'finished':
msg = '<br />Extracting audio (this can take a while)...\n'
elif status == 'error':
# we always get an 'error' progress when the video finishes downloading.
# not sure why. ignore it.
return
elif status == 'downloading':
p = lambda field: progress.get(field) or ''
try:
percent = float(p('_percent_str').strip('%') or '0')
except ValueError:
percent = 0
msg = ('<span><progress max="100" value="%s"></progress><br /> '
'%s of %s at %s in %s...</span>\n' % (
percent, p('_downloaded_bytes_str'),
p('_total_bytes_str') or p('_total_bytes_estimate_str'),
p('_speed_str'), p('_eta_str')))
else:
msg = status + '<br />\n'
write(msg)
# fetch video info (resolves URL) to see if we've already downloaded it
options = {
'outtmpl': u'/tmp/%(webpage_url)s',
'restrictfilenames': True, # don't allow & or spaces in file names
'updatetime': False, # don't set output file mtime to video mtime
'logger': logging,
'logtostderr': True,
'format': 'bestaudio/best',
'noplaylist': True,
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'progress_hooks': [download_progress_hook],
}
ydl = youtube_dl.YoutubeDL(options)
with handle_errors(write):
info = ydl.extract_info(url, download=False)
# prepare_filename() returns the video filename, not the postprocessed one,
# so change the extension manually. the resulting filename will look like:
# '/tmp/https_-_www.youtube.com_watchv=6dyWlM4ej3Q.mp3'
#
# ext4 max filename length is 255 bytes, and huffduffer also silently
# truncates URLs to 255 chars total, so truncate before that if necessary.
filename_prefix = ydl.prepare_filename(info)[:245 - len(B2_BASE)]
options['outtmpl'] = filename_prefix.replace('%', '%%') + '.%(ext)s'
filename = filename_prefix + '.mp3'
b2api = B2Api(InMemoryAccountInfo())
b2api.authorize_account('production', B2_KEY_ID, B2_APP_KEY)
bucket = b2api.get_bucket_by_name(B2_BUCKET)
# strip the filename's path, scheme, and leading www., mobile, or m.
# the resulting filename will look like 'youtube.com_watchv=6dyWlM4ej3Q.mp3'
b2_filename = re.sub('^https?_-_((www|m|mobile|player).)?', '', os.path.basename(filename))
b2_url = bucket.get_download_url(b2_filename)
uploaded_time = datetime.datetime.now()
existing = requests.head(b2_url)
if existing.ok:
yield 'Already downloaded! <br />\n'.encode()
try:
uploaded_time = datetime.datetime.utcfromtimestamp(
int(existing.headers.get('X-Bz-Upload-Timestamp')) / 1000) # ms
except:
# missing or bad header
pass
else:
# download video and extract mp3
yield 'Downloading (this can take a while)...<br />\n'.encode()
with handle_errors(write):
youtube_dl.YoutubeDL(options).download([url])
# upload to B2
yield 'Uploading to B2...<br />\n'.encode()
class WriteProgress(AbstractProgressListener):
def set_total_bytes(self, total):
self.total = total
def bytes_completed(self, sent):
write('<span><progress max="100" value="%s"></progress><br /> '
'%.2fMB of %.2fMB</span>\n' % (
(sent * 100 / self.total), float(sent) / 1000000,
float(self.total) / 1000000))
def close(self):
pass
with WriteProgress() as listener:
bucket.upload_local_file(filename, b2_filename, progress_listener=listener)
os.remove(filename)
# generate description
description = info.get('description') or ''
footer = """\
Original video: %s
Downloaded by http://huffduff-video.snarfed.org/ on %s
Available for 30 days after download""" % (
url, uploaded_time.replace(microsecond=0).ctime())
if description:
footer = """
===
""" + footer
max_len = 1500 - len(footer)
if len(description) > max_len:
description = description[:max_len] + '...'
description += footer
# open 'Huffduff it' page
yield ("""\n<br />Opening Huffduffer dialog...
<script type="text/javascript">
window.location = "https://huffduffer.com/add?popup=true&%s";
</script>
""" % urllib.parse.urlencode([(k, v.encode()) for k, v in
(('bookmark[url]', (b2_url)),
('bookmark[title]', info.get('title') or ''),
('bookmark[description]', description),
('bookmark[tags]', ','.join(info.get('categories') or [])),
)])).encode()
yield HTML_FOOTER.encode()
def __init__(self, arg):
"""Initialize B2 connection and credentials."""
info = SqliteAccountInfo() # creds and tokens in ~/.b2_account_info
b2_api = B2Api(info)
self.arg = arg
