def test_humanizeBytes():
for (input_bytes, expected_output) in [
(-1, '-1.0bytes'),
(0, '0.0bytes'),
(1, '1.0bytes'),
(10, '10.0bytes'),
((2**10) - 1, '1023.0bytes'),
((2**10), '1.0kB'),
((2**20), '1.0MB'),
((2**20) * 1.5, '1.5MB'),
((2**70), 'Oops larger than Exabytes'),
]:
assert utils.humanizeBytes(input_bytes) == expected_output
def printTransferProgress(self,
transferred,
toBeTransferred,
prefix='',
postfix='',
isBytes=True,
dt=None,
previouslyTransferred=0):
"""
Parameters match those of synapseclient.core.utils.printTransferProgress.
"""
if not sys.stdout.isatty():
return
with self._lock:
if toBeTransferred == 0 or float(
transferred) / toBeTransferred >= 1:
# if the individual transfer is complete then we pass through the print
# to the underlying utility method which will print a complete 100%
# progress bar on a newline.
utils.printTransferProgress(
transferred,
toBeTransferred,
prefix=prefix,
postfix=postfix,
isBytes=isBytes,
dt=dt,
previouslyTransferred=previouslyTransferred)
# in order to know how much of the transferred data is newly transferred
# we subtract the previously reported amount. this assumes that the printing
# of the progress for any particular transfer is always conducted by the same
# thread, which is true for all current transfer implementations.
self._total_transferred += (transferred -
_thread_local.thread_transferred)
_thread_local.thread_transferred = transferred
cumulative_dt = time.time() - self._start
rate = self._total_transferred / float(cumulative_dt)
rate = '(%s/s)' % utils.humanizeBytes(rate) if isBytes else rate
# we print a rotating tick with each update
self._spinner.print_tick()
sys.stdout.write(
f"{self._label} {utils.humanizeBytes(self._total_transferred)} {rate}"
)
sys.stdout.flush()
def test_humanizeBytes__None():
with pytest.raises(ValueError):
utils.humanizeBytes(None)
def syncToSynapse(syn,
manifestFile,
dryRun=False,
sendMessages=True,
retries=MAX_RETRIES):
"""Synchronizes files specified in the manifest file to Synapse
:param syn: A synapse object as obtained with syn = synapseclient.login()
:param manifestFile: A tsv file with file locations and metadata to be pushed to Synapse.
See below for details
:param dryRun: Performs validation without uploading if set to True (default is False)
Given a file describing all of the uploads uploads the content to Synapse and optionally notifies you via Synapse
messagging (email) at specific intervals, on errors and on completion.
**Manifest file format**
The format of the manifest file is a tab delimited file with one row per file to upload and columns describing the
file. The minimum required columns are **path** and **parent** where path is the local file path and parent is the
Synapse Id of the project or folder where the file is uploaded to. In addition to these columns you can specify any
of the parameters to the File constructor (**name**, **synapseStore**, **contentType**) as well as parameters to the
syn.store command (**used**, **executed**, **activityName**, **activityDescription**, **forceVersion**).
Used and executed can be semi-colon (";") separated lists of Synapse ids, urls and/or local filepaths of files
already stored in Synapse (or being stored in Synapse by the manifest).
Any additional columns will be added as annotations.
**Required fields:**
====== ====================== ============================
Field Meaning Example
====== ====================== ============================
path local file path or URL /path/to/local/file.txt
parent synapse id syn1235
====== ====================== ============================
**Common fields:**
=============== =========================== ============
Field Meaning Example
=============== =========================== ============
name name of file in Synapse Example_file
forceVersion whether to update version False
=============== =========================== ============
**Provenance fields:**
==================== ===================================== ==========================================
Field Meaning Example
==================== ===================================== ==========================================
used List of items used to generate file syn1235; /path/to_local/file.txt
executed List of items exectued https://github.org/; /path/to_local/code.py
activityName Name of activity in provenance "Ran normalization"
activityDescription Text description on what was done "Ran algorithm xyx with parameters..."
==================== ===================================== ==========================================
Annotations:
**Annotations:**
Any columns that are not in the reserved names described above will be interpreted as annotations of the file
**Other optional fields:**
=============== ========================================== ============
Field Meaning Example
=============== ========================================== ============
synapseStore Boolean describing whether to upload files True
contentType content type of file to overload defaults text/html
=============== ========================================== ============
**Example manifest file**
=============== ======== ======= ======= =========================== ============================
path parent annot1 annot2 used executed
=============== ======== ======= ======= =========================== ============================
/path/file1.txt syn1243 "bar" 3.1415 "syn124; /path/file2.txt" "https://github.org/foo/bar"
/path/file2.txt syn12433 "baz" 2.71 "" "https://github.org/foo/baz"
=============== ======== ======= ======= =========================== ============================
"""
df = readManifestFile(syn, manifestFile)
# have to check all size of single file
sizes = [
os.stat(os.path.expandvars(os.path.expanduser(f))).st_size
for f in df.path if not is_url(f)
]
# Write output on what is getting pushed and estimated times - send out message.
sys.stdout.write('=' * 50 + '\n')
sys.stdout.write(
'We are about to upload %i files with a total size of %s.\n ' %
(len(df), utils.humanizeBytes(sum(sizes))))
sys.stdout.write('=' * 50 + '\n')
if dryRun:
return
sys.stdout.write('Starting upload...\n')
if sendMessages:
notify_decorator = notifyMe(syn,
'Upload of %s' % manifestFile,
retries=retries)
upload = notify_decorator(_manifest_upload)
upload(syn, df)
else:
_manifest_upload(syn, df)