def downloadMetaData(self, simulateDownload=False):
""" Download metadata """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
self.d.downTime = timezone.now()
# get root directory
root = self.service.metadata("/",
include_deleted=True,
include_media_info=True)
# get others directory
fileMetaData = self.recurseDropTree(root, 5)
self.metadata = fileMetaData
meta = base64.b64encode(json.dumps(fileMetaData))
metaTime = timezone.now()
metaHash = crypto.rsaSignatureSHA256(
meta + crypto.HASH_SEPARATOR + format(metaTime, "U"),
settings.PRIV_KEY)
#store the data
storeFM = FileMetadata(metadata=meta,
tokenID=self.t,
metaTime=metaTime,
metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def downloadMetaData(self,simulateDownload = False):
""" Download metadata """
#used for test
if simulateDownload is True:
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
self.d.downTime = timezone.now()
# get root directory
root = self.service.metadata("/",include_deleted=True,include_media_info=True)
# get others directory
fileMetaData = self.recurseDropTree(root,5)
self.metadata = fileMetaData
meta = base64.b64encode(json.dumps(fileMetaData))
metaTime = timezone.now()
metaHash = crypto.rsaSignatureSHA256(meta+crypto.HASH_SEPARATOR+format(metaTime,"U"),settings.PRIV_KEY)
#store the data
storeFM = FileMetadata(metadata=meta,tokenID=self.t,metaTime=metaTime,metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def downloadMetaData(self,simulateDownload = False):
""" Download the metadata """
#used for tests
if simulateDownload is True:
#simulate the download by waiting 10 seconds
time.sleep(constConfig.TEST_THREAD_SLEEP_TIME)
return downStatus
#download
self.d.downTime = timezone.now()
result = []
page_token = None
while True:
param = {}
if page_token:
param['pageToken'] = page_token
param['maxResults'] = 500
files = self.service.files().list(**param).execute()
result.extend(files['items'])
page_token = files.get('nextPageToken')
if not page_token:
break
self.metadata = result
meta = base64.b64encode(json.dumps(self.metadata))
metaTime = timezone.now()
txt = meta+crypto.HASH_SEPARATOR+format(metaTime,"U")
metaHash = crypto.rsaSignatureSHA256(txt,settings.PRIV_KEY)
storeFM = FileMetadata(metadata=meta,tokenID=self.t,metaTime=metaTime,metadataHash=metaHash)
storeFM.save()
self.d.threadStatus = constConfig.THREAD_DOWN_META
self.d.save()
def get(self):
user = users.get_current_user()
username = user.nickname()
logout_url = users.create_logout_url('/')
upload_url = blobstore.create_upload_url("/upload")
first = FileMetadata.getFirstKeyForUser(username)
last = FileMetadata.getLastKeyForUser(username)
q = FileMetadata.all()
q.filter('__key__ >', first)
q.filter('__key__ <', last)
results = q.fetch(10)
items = [_ for _ in results]
length = len(items)
self.response.out.write(self.template_env.get_template('index.html').render(
{
'username': username,
'logout_url': logout_url,
'upload_url': upload_url,
'items': items,
'length': length
}
))
def post(self):
upload_files = self.get_uploads('file')
blob_key = upload_files[0].key()
name = self.request.get('name')
user = users.get_current_user()
username = user.nickname()
date = datetime.datetime.now()
str_blob_key = str(blob_key)
key = FileMetadata.getKeyName(username, date, str_blob_key)
m = FileMetadata(key_name=key)
m.filename = name
m.uploadedOn = date
m.owner = user
m.blobkey = str_blob_key
m.put()
self.redirect('/')
def process_license_files(current_file, file_name, file_metadata_dict):
license_array = []
copyright_array = []
licenses = current_file['licenses']
for lic in licenses:
license_array.append(lic['spdx_license_key'])
copyrights = current_file['copyrights']
for cprt in copyrights:
# holders = "\n".join(cprt['holders'])
holders = cprt['holders']
for holder in holders:
copyright_array.append(holder)
file_metadata = FileMetadata(" ".join(set(copyright_array)),
" ".join(set(license_array)))
file_metadata_dict[file_metadata] = {"N/A": [file_name]}
def process_copyright_and_license_information(scancode_output_file):
f = open(scancode_output_file, "r")
if f.mode == 'r':
file_contents = f.read()
parsed_json = json.loads(file_contents)
files = parsed_json['files']
file_metadata_dict = {}
for current_file in files:
file_name = current_file['path']
file_ext = current_file['extension']
if not re.match("license",
str(file_name).split("/")[-1], re.IGNORECASE):
if len(file_ext) == 0:
file_ext = "N/A"
# Most of the file has got just one copyright. To get unique combination of license +
# copyright # ,get the first copyright information from copyright array
try:
copyrights = current_file['copyrights']
if len(copyrights) != 0:
cprt = copyrights[0]
try:
holders = cprt['holders']
try:
if len(holders) != 0:
cprt_holder = holders[0]
else:
cprt_holder = "N/A"
except KeyError:
cprt_holder = "N/A"
except KeyError:
cprt_holder = "N/A"
else:
cprt_holder = "N/A"
except KeyError:
cprt_holder = "N/A"
try:
licenses = current_file['licenses']
# iterate over array of licenses for this file
if len(licenses) != 0:
for lic in licenses:
file_metadata = FileMetadata(
cprt_holder, lic['spdx_license_key'])
if file_metadata not in file_metadata_dict:
file_ext_dic = {file_ext: [file_name]}
file_metadata_dict[
file_metadata] = file_ext_dic
else:
existing_file_ext_dic = file_metadata_dict.get(
file_metadata)
if file_ext in existing_file_ext_dic:
contained_files = existing_file_ext_dic[
file_ext]
# print(contained_files)
if file_name not in contained_files:
contained_files.append(file_name)
else:
existing_file_ext_dic[file_ext] = [
file_name
]
else:
file_metadata = FileMetadata(cprt_holder, "N/A")
if file_metadata not in file_metadata_dict:
file_ext_dic = {file_ext: [file_name]}
file_metadata_dict[file_metadata] = file_ext_dic
else:
existing_file_ext_dic = file_metadata_dict.get(
file_metadata)
if file_ext in existing_file_ext_dic:
contained_files = existing_file_ext_dic[
file_ext]
# print(contained_files)
if file_name not in contained_files:
contained_files.append(file_name)
else:
existing_file_ext_dic[file_ext] = [file_name]
except KeyError:
file_metadata = FileMetadata(cprt_holder, "N/A")
if file_metadata not in file_metadata_dict:
file_ext_dic = {file_ext: [file_name]}
file_metadata_dict[file_metadata] = file_ext_dic
else:
existing_file_ext_dic = file_metadata_dict.get(
file_metadata)
if file_ext in existing_file_ext_dic:
contained_files = existing_file_ext_dic[file_ext]
# print(contained_files)
if file_name not in contained_files:
contained_files.append(file_name)
else:
existing_file_ext_dic[file_ext] = [file_name]
else:
process_license_files(current_file, file_name,
file_metadata_dict)
# print_file_metadata(file_metadata_dict)
return file_metadata_dict
