def run_inference(im, model, result_code): # Perform inference inference_start = time.time() result = model.run_inference(im)[0] inference_duration = time.time() - inference_start result = cv2.cvtColor((result * 255).astype(np.float32), cv2.COLOR_BGR2RGB) # Save image pink_path = f"results/{result_code}.jpg" cv2.imwrite(pink_path, result) # Upload to b2 upload_start = time.time() info = InMemoryAccountInfo() b2_api = B2Api(info) b2_api.authorize_account("production", config.B2_KEY, config.B2_SECRET) bucket = b2_api.get_bucket_by_name(config.B2_BUCKET) bucket.upload_local_file( local_file=pink_path, file_name=f"cycleganime/results/{os.path.basename(pink_path)}") upload_duration = time.time() - upload_start # Remove local file os.remove(pink_path) # Log output with open("log.txt", "a") as f: f.write( f"{pink_path}\t{inference_duration:.2f}\t{upload_duration:.2f}\n")
def sync(): """Synchronize files modified in last x days from the bucket to the media folder""" b2 = B2Api(InMemoryAccountInfo()) app_key_id = get_snap_config('b2-application-key-id') app_key = get_snap_config('b2-application-key') b2.authorize_account('production', app_key_id, app_key) exclude_before_timestamp = int(time.time()) - (int(get_snap_config('remove-after-days')) * 86400) policies_manager = ScanPoliciesManager( exclude_file_regexes=("index.html",), exclude_modified_before=exclude_before_timestamp * 1000 # in ms ) synchronizer = Synchronizer( max_workers=5, newer_file_mode=NewerFileSyncMode.SKIP, policies_manager=policies_manager ) bucket_uri = 'b2://' + get_snap_config('b2-bucket') source = parse_sync_folder(bucket_uri, b2) destination = parse_sync_folder(MEDIA_DIR, b2) if not os.path.isdir(MEDIA_DIR): os.makedirs(MEDIA_DIR) with SyncReport(sys.stdout, False) as reporter: synchronizer.sync_folders( source_folder=source, dest_folder=destination, now_millis=time.time() * 1000, # in ms reporter=reporter )
def b2Api(self) -> B2Api: if not hasattr(self, "_b2Api"): self._accountInfo = (SqliteAccountInfo( file_name=self._sqliteDbPath) if hasattr( self, "_sqliteDbPath") else InMemoryAccountInfo()) self._b2Api = B2Api(self._accountInfo) self._b2Api.authorize_account(**self._authInfo) return self._b2Api
def __init__(self, application_key_id, application_key, bucket_name): """ Initialize b2_api and authenticate account """ info = InMemoryAccountInfo() self.b2_api = B2Api(info) self.application_key_id = application_key_id self.application_key = application_key self.b2_api.authorize_account("production", application_key_id, application_key) self.bucket = self.b2_api.get_bucket_by_name(bucket_name)
def __init__(self, account_id, application_key, bucket_name_prefix): self.account_id = account_id self.application_key = application_key self.bucket_name_prefix = bucket_name_prefix info = InMemoryAccountInfo() cache = InMemoryCache() self.api = B2Api(info, cache=cache) self.api.authorize_account('production', self.account_id, self.application_key)
def upload_guide(): """Upload the "all.xml" file to BackBlaze B2""" # Initialize BackBlaze B2 API info = InMemoryAccountInfo() api = B2Api(info) # Login to BackBlaze api.authorize_account("production", B2_ID, B2_KEY) bucket = api.get_bucket_by_name(B2_BUCKET) # Upload the "all.xml" file, which contain the tv guide bucket.upload_local_file(local_file="all.xml", file_name="all.xml") print("[+] Guide upload completed")
def run(): if "B2_KEY_ID" not in os.environ.keys(): logging.error("Please set environment variables: B2_KEY_ID and B2_KEY") sys.exit(1) B2_KEY_ID = os.environ['B2_KEY_ID'] B2_KEY = os.environ['B2_KEY'] B2_BUCKET_NAME = os.environ['B2_BUCKET_NAME'] args = parse_args() logging.debug("Args: {}".format(args)) screenshotFolder = os.path.abspath(args.directory) logging.debug("Configured screenshotsFolder: {}".format(args.directory)) if not os.path.isdir(screenshotFolder): logging.error("File {screenshotFolder} doesn't exist!") exit(1) # Initialize B2Api info = InMemoryAccountInfo() b2_api = B2Api(info) b2_api.authorize_account("production", B2_KEY_ID, B2_KEY) bucket = b2_api.get_bucket_by_name(B2_BUCKET_NAME) # build random word string for filename r = RandomWord() c = ["the", "my", "his", "her", "our", "their", "that", "this"] randwords = '-'.join([ r.word(include_parts_of_speech=["verbs"]), random.choice(c), r.word(include_parts_of_speech=["adjectives"]), r.word(include_parts_of_speech=["nouns"]) ]) fileName = "{}.png".format(randwords) logging.debug("Filename {}".format(fileName)) fullFile = os.path.join(screenshotFolder, fileName) logging.debug("Full file path: {}".format(fullFile)) if args.area: take_area_screenshot(fullFile) if args.window: take_window_screenshot(fullFile) if args.screen: take_screenshot(fullFile) uploadResults = bucket.upload_local_file(local_file=fullFile, file_name=fileName) fileId = uploadResults.as_dict()['fileId'] print(b2_api.get_download_url_for_file_name(B2_BUCKET_NAME, fileName))
def upload( model: str, directory: str, bucket: str = 'malaya', application_key_id: str = os.environ.get('backblaze_application_key_id'), application_key: str = os.environ.get('backblaze_application_key')): """ Upload directory with malaya-style pattern. Parameters ---------- model: str it will become directory name. directory: str local directory with files in it. bucket: str, optional (default='malaya') backblaze bucket. application_key_id: str, optional (default=os.environ.get('backblaze_application_key_id')) application_key: str, optional (default=os.environ.get('backblaze_application_key')) """ if not application_key_id or not application_key: raise ValueError( 'must set `backblaze_application_key_id` and `backblaze_application_key` are None.' ) from b2sdk.v1 import B2Api, InMemoryAccountInfo info = InMemoryAccountInfo() b2_api = B2Api(info) b2_api.authorize_account('production', application_key_id, application_key) file_info = {'how': 'good-file'} b2_bucket = b2_api.get_bucket_by_name(bucket) for file in glob(os.path.join(directory, '*')): if file.endswith('frozen_model.pb'): outPutname = f'{model}/model.pb' elif file.endswith('frozen_model.pb.quantized'): outPutname = f'{model}-quantized/model.pb' else: outPutname = f'{model}/{file}' b2_bucket.upload_local_file( local_file=file, file_name=outPutname, file_infos=file_info, ) logger.info(f'Uploaded from local {file} to {bucket}/{outPutname}')
def upload_epaper_to_backblaze(pdf_info: dict, app_key_id: str, app_key: str, bucket_name: str): info: InMemoryAccountInfo = InMemoryAccountInfo() b2_api: B2Api = B2Api(info) b2_api.authorize_account("production", app_key_id, app_key) bucket = b2_api.get_bucket_by_name(bucket_name) print(f'Uploading {pdf_info["cover_file_path"]}') bucket.upload_local_file( local_file=pdf_info["cover_file_path"], file_name=f'cover_{pdf_info["edition"]}.jpg', ) print(f'Uploading {pdf_info["epaper_file_path"]}') bucket.upload_local_file(local_file=pdf_info["epaper_file_path"], file_name=f'{pdf_info["edition"]}.pdf') print("Files uploaded.")
def api(self): ''' The B2API, authorized from `self.credentials`. ''' api = B2Api(InMemoryAccountInfo()) # monkey patch the API instance to serialise reauthorization attempts b2authorize_account = api.authorize_account def locked_authorize_account(*a): ''' Serialised version of the API authorize_account method. ''' with self._lock: return b2authorize_account(*a) api.authorize_account = locked_authorize_account with self._conn_sem: api.authorize_account("production", self.credentials.keyId, self.credentials.apiKey) return api
def load_result_ids(): """Returns set of file ids and a queue ordering them""" result_ids = set() info = InMemoryAccountInfo() b2_api = B2Api(info) b2_api.authorize_account("production", config.B2_KEY, config.B2_SECRET) bucket = b2_api.get_bucket_by_name(config.B2_BUCKET) files = [] for file_info, _ in bucket.ls(folder_to_list="cycleganime/results", show_versions=False, recursive=True): if file_info.content_type != "image/jpeg": continue result_id = os.path.basename(file_info.file_name)[:5] upload_timestamp = file_info.upload_timestamp result_ids.add(result_id) files.append((result_id, upload_timestamp)) files = sorted(files, key=lambda x: x[1]) # Sort by upload timestamp result_q = deque(files) return result_ids, result_q
def b2_download_models(): # Retrieve all relevant credentials / info from environment application_key_id = env['B2_APPLICATION_ID'] application_key = env['B2_APPLICATION_KEY'] application_realm = env['B2_APPLICATION_REALM'] bucket_name = env['B2_BUCKET_NAME'] # Authenticate using credentials info = InMemoryAccountInfo( ) # store credentials, tokens and cache in memory b2_api = B2Api(info) b2_api.authorize_account(application_realm, application_key_id, application_key) bucket = b2_api.get_bucket_by_name(bucket_name) subreddit_models_dir = 'subreddit-clfs' norm_models_dir = 'norm-clfs' directory_prefix = f'tmp/backend-ml/{subreddit_models_dir}' for directory in [subreddit_models_dir, norm_models_dir]: full_directory = path.join(directory_prefix, directory) if not path.exists(full_directory): makedirs(full_directory) subreddits = \ [ "Futurology", "science" ] for subreddit in subreddits: for extension in [".vec", ".bin"]: b2_filename = f'{subreddit_models_dir}/model_{subreddit}{extension}' local_file_path = f'{directory_prefix}/model_{subreddit}{extension}' if path.exists(local_file_path): print(f'Model {extension} for subreddit {subreddit} exists!') continue download_dest = DownloadDestLocalFile(local_file_path) progress_listener = SimpleProgressListener( f'Downloading model: model_{subreddit}{extension}') bucket.download_file_by_name(b2_filename, download_dest, progress_listener)
def b2_upload(bucket_name, filename): from b2sdk.v1 import InMemoryAccountInfo from b2sdk.v1 import B2Api info = InMemoryAccountInfo() b2 = B2Api(info) app_key_id = "William" app_key = "Gibson" b2.authorize_account("production", app_key_id, app_key) # Upload a single file local_file_path = filename b2_file_name = filename file_info = {"how": "good-file"} bucket = b2.get_bucket_by_name(bucket_name) bucket.upload_local_file( local_file=local_file_path, file_name=b2_file_name, file_infos=file_info, )
def __init__(self): log("Initializing Clipper") self.clips: Dict[AnyStr, Clip] = None self.c = load_yaml() logging.basicConfig( level=self.c["Logging"]["level"], format=self.c["Logging"]["format"] ) self.meta = ClipsMeta.from_url( self.get_config("MetaSource", "endpoint"), self.get_config("MetaSource", "token") ) self.load_local_clips() key_id = self.get_config("B2", "key_id") app_key = self.get_config("B2", "app_key") log("Initializing B2") info = InMemoryAccountInfo() self._api = B2Api(info) self._api.authorize_account("production", key_id, app_key) self._bucket = self._api.get_bucket_by_name( self.get_config("B2", "bucket_name") ) self._file_link_template = "https://rushia.moe/clips?uid={uid}" log("Done initializing Clipper")
def __init__(self, model, db_session=None): self.api = B2Api(InMemoryAccountInfo()) self.db = db_session self.model = model
def __init__(self, replica): self.app_key_id = replica['meta']['app_key_id'] self.app_key = replica['meta']['app_key'] info = InMemoryAccountInfo() self.api = B2Api(info) self.bucket = 'openpacs'
def _authorize(self, keyid: str, appkey: str, bucketname: str): b2_api = B2Api(InMemoryAccountInfo()) b2_api.authorize_account("production", keyid, appkey) bucket = b2_api.get_bucket_by_name(bucketname,) return b2_api,bucket
def list_target_files(config): import urllib.parse try: target = urllib.parse.urlparse(config["target"]) except ValueError: return "invalid target" if target.scheme == "file": return [(fn, os.path.getsize(os.path.join(target.path, fn))) for fn in os.listdir(target.path)] elif target.scheme == "rsync": rsync_fn_size_re = re.compile(r'.* ([^ ]*) [^ ]* [^ ]* (.*)') rsync_target = '{host}:{path}' target_path = target.path if not target_path.endswith('/'): target_path = target_path + '/' if target_path.startswith('/'): target_path = target_path[1:] rsync_command = [ 'rsync', '-e', '/usr/bin/ssh -i /root/.ssh/id_rsa_miab -oStrictHostKeyChecking=no -oBatchMode=yes', '--list-only', '-r', rsync_target.format(host=target.netloc, path=target_path) ] code, listing = shell('check_output', rsync_command, trap=True, capture_stderr=True) if code == 0: ret = [] for l in listing.split('\n'): match = rsync_fn_size_re.match(l) if match: ret.append((match.groups()[1], int(match.groups()[0].replace(',', '')))) return ret else: if 'Permission denied (publickey).' in listing: reason = "Invalid user or check you correctly copied the SSH key." elif 'No such file or directory' in listing: reason = "Provided path {} is invalid.".format(target_path) elif 'Network is unreachable' in listing: reason = "The IP address {} is unreachable.".format( target.hostname) elif 'Could not resolve hostname' in listing: reason = "The hostname {} cannot be resolved.".format( target.hostname) else: reason = "Unknown error." \ "Please check running 'management/backup.py --verify'" \ "from mailinabox sources to debug the issue." raise ValueError( "Connection to rsync host failed: {}".format(reason)) elif target.scheme == "s3": # match to a Region fix_boto() # must call prior to importing boto import boto.s3 from boto.exception import BotoServerError custom_region = False for region in boto.s3.regions(): if region.endpoint == target.hostname: break else: # If region is not found this is a custom region custom_region = True bucket = target.path[1:].split('/')[0] path = '/'.join(target.path[1:].split('/')[1:]) + '/' # Create a custom region with custom endpoint if custom_region: from boto.s3.connection import S3Connection region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection) # If no prefix is specified, set the path to '', otherwise boto won't list the files if path == '/': path = '' if bucket == "": raise ValueError("Enter an S3 bucket name.") # connect to the region & bucket try: conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"]) bucket = conn.get_bucket(bucket) except BotoServerError as e: if e.status == 403: raise ValueError("Invalid S3 access key or secret access key.") elif e.status == 404: raise ValueError("Invalid S3 bucket name.") elif e.status == 301: raise ValueError("Incorrect region for this bucket.") raise ValueError(e.reason) return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)] elif target.scheme == 'b2': from b2sdk.v1 import InMemoryAccountInfo, B2Api from b2sdk.v1.exception import NonExistentBucket info = InMemoryAccountInfo() b2_api = B2Api(info) # Extract information from target b2_application_keyid = target.netloc[:target.netloc.index(':')] b2_application_key = target.netloc[target.netloc.index(':') + 1:target.netloc.index('@')] b2_bucket = target.netloc[target.netloc.index('@') + 1:] try: b2_api.authorize_account("production", b2_application_keyid, b2_application_key) bucket = b2_api.get_bucket_by_name(b2_bucket) except NonExistentBucket as e: raise ValueError( "B2 Bucket does not exist. Please double check your information!" ) return [(key.file_name, key.size) for key, _ in bucket.ls()] else: raise ValueError(config["target"])
def get_bucket(config): b2_config = config['b2'] account_info = InMemoryAccountInfo() b2_api = B2Api(account_info=account_info) b2_api.authorize_account('production', b2_config['key'], b2_config['secret']) return b2_api.get_bucket_by_name(b2_config['bucket'])
def replace_keys(): """Replace the placeholder authorization token and upload URL in the index file""" with open(INDEX_FILE, 'r+') as f: data = f.read() f.seek(0) t = tokens() replaced = data.replace('CHANGEME_AUTHORIZATION_TOKEN', t['authorizationToken']) replaced = replaced.replace('CHANGEME_UPLOAD_URL', t['uploadUrl']) f.write(replaced) f.truncate() def upload(): """Upload the modified file to the bucket""" name = get_snap_config('b2-bucket') bucket = B2.get_bucket_by_name(name) bucket.upload_local_file(INDEX_FILE, 'index.html') B2 = B2Api(InMemoryAccountInfo()) app_key_id = get_snap_config('b2-application-key-id') app_key = get_snap_config('b2-application-key') B2.authorize_account('production', app_key_id, app_key) copy_template() replace_keys() upload()
import json from b2_browser.settings import B2_API_ID, B2_API_KEY, B2_BUCKET, DEBUG from b2sdk.v1 import InMemoryAccountInfo from b2sdk.v1 import B2Api from django.core.cache import cache import datetime import os _info = InMemoryAccountInfo() _api_obj = B2Api(_info) _api_obj.authorize_account('production', B2_API_ID, B2_API_KEY) def b2ls(path=''): a = cache.get(f'path:{path}') if a is not None: return json.loads(a) # Get a list of file / folders obj = [] b = _api_obj.get_bucket_by_name(B2_BUCKET) for f_info, fn in b.ls(show_versions=False, folder_to_list=path, recursive=False): entry = {} if fn is not None:
# Load up JSON so we can grab a message with open(os.getenv("GITHUB_EVENT_PATH")) as fh: event_data = json.load(fh) if GITHUB_EVENT_NAME == "push": # Commit name HEAD_MESSAGE = event_data['head_commit']['message'] else: HEAD_MESSAGE = "" if __name__ == "__main__": src_dir = sys.argv[1] dest_dir = sys.argv[2] info = InMemoryAccountInfo() b2_api = B2Api(info) b2_api.authorize_account("production", B2_KEY_ID, B2_APP_KEY) bucket = b2_api.get_bucket_by_id(B2_BUCKET_ID) # Iterate every file in the directory. for filename in pathlib.Path(src_dir).iterdir(): print(f"==> Uploading {filename}...") bucket.upload_local_file( filename, dest_dir + "/" + filename.name, content_type="application/zip", file_infos={ "platform": dest_dir, "event_name": GITHUB_EVENT_NAME, "sha": GITHUB_SHA, "ref": GITHUB_REF,
def __init__(self, name: str, keys: list): self.api = B2Api(info) self.name = name self.key, self.key_id = keys self.connected = False