def enhancePages( self, project, connector_url, cq_user, cq_password, page_query, page_queryLib ):
# Validate Parameters
if not self.validInputs(connector_url,cq_user,cq_password,"asset_drive","asset_filenameAttr","asset_pathAttr",page_query,page_queryLib,"ia_path","ia_cacheCols"): return
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password))
header = Header("Authorization", "Basic %s" % (base64EncodedCredentials))
# Cycle through contentdescriptors specified by scope query
query = queryLibrarianService.findQueryByName(page_queryLib, page_query)
params = HashMap()
params.put('projectId',project.id)
res = queryLibrarianService.executeQuery(query, params)
# Get cd
for r in res:
cd = r[0]
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
try:
payload = cd.getContent()[0].getContentData().encode("utf-8")
[loadStatus, failureReason] = self.sendPayload(client,get,post,payload,cd.url)
cd.metadata["Load.Status"] = loadStatus
cd.metadata["Load.Failure Reason"] = failureReason
if loadStatus != "SUCCESS":
logger.error("url: %s, reason: %s" % (cd.url, failureReason))
except:
cd.metadata["Load.Status"] = "FAILURE"
cd.metadata["Load.Failure Reason"] = "Null content"
logger.error("url: %s, reason: %s" % (cd.url, failureReason))
def __get(self):
try:
get = GetMethod(self.url)
statusInt = client.executeMethod(get)
r = str(statusInt), get.getResponseBodyAsString().strip()
except Exception, e:
r = str(e), None
def enhanceIA( self, connector_url, cq_user, cq_password, ia_path, ia_cacheCols ):
# Validate Parameters
if not self.validInputs(connector_url,cq_user,cq_password,"asset_drive","asset_filenameAttr","asset_pathAttr","page_query","page_queryLib",ia_path,ia_cacheCols): return
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password))
header = Header("Authorization", "Basic %s" % (base64EncodedCredentials))
# IA Build
cached_IA = self.cacheIA(ia_path,ia_cacheCols)
for placeholder in cached_IA:
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
# Prep Payload
path = placeholder["Path"]
filename = placeholder["Filename"]
template_path = placeholder["Template Path"]
label = placeholder["Label"]
# Send Payload
payload = '<page path="%s" filename="%s" template_path="%s" label="%s"><node path="jcr:content" type="nt:unstructured"><prop name="jcr:title" type="String">%s</prop></node></page>' % (path,filename,template_path,label,label)
[loadStatus, failureReason] = self.sendPayload(client,get,post,payload,path+filename)
if loadStatus != "SUCCESS":
logger.error("url: %s, reason: %s" % (placeholder["URL"], failureReason))
def updateLocalRecordRelations(self, jobItems):
oidIdentifierMap = HashMap()
for jobItem in jobItems:
oidIdentifierMap.put(jobItem.get("oid"),jobItem.get("required_identifiers")[0].get("identifier"))
for jobItem in jobItems:
type = jobItem.get("type");
targetSystem = self.systemConfig.getString(None, "curation", "supported-types", type);
if targetSystem == "redbox":
oid = jobItem.get("oid")
digitalObject = StorageUtils.getDigitalObject(self.services.getStorage(), oid)
tfPackagePid = self.getPackageData(digitalObject)
metadataJsonPayload = digitalObject.getPayload(tfPackagePid)
metadataJsonInstream = metadataJsonPayload.open()
metadataJson = JsonSimple(metadataJsonInstream)
metadataJsonPayload.close()
relationships = metadataJson.getArray("relationships")
if relationships is not None:
for relationship in relationships:
system = relationship.get("system")
if system != "redbox" or system != None:
url = self.systemConfig.getString("can't find it", "curation","external-system-urls","get-oid-for-identifier",system)
client = BasicHttpClient(url+ "&identifier="+relationship.get("identifier"))
get = GetMethod(url+ "&identifier="+relationship.get("identifier"))
client.executeMethod(get)
if get.getStatusCode() == 200:
response = JsonSimple(get.getResponseBodyAsString())
relationship.put("curatedPid",oidIdentifierMap.get(response.getString(None,"oid")))
relationship.put("isCurated",True)
#Now update the relationship on Mint's side
break
istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes())
StorageUtils.createOrUpdatePayload(digitalObject,tfPackagePid,istream)
def __get(self):
try:
get = GetMethod(self.url)
statusInt = self.client.executeMethod(get)
r = str(statusInt), get.getResponseBodyAsString().strip()
except Exception, e:
r = str(e), None
def __get(self, url):
try:
client = BasicHttpClient(url)
get = GetMethod(url)
statusInt = client.executeMethod(get)
s = get.getResponseBodyAsString().strip()
except Exception, e:
print "__get() Error - '%s'" % str(e)
s = ""
def urlGet(self, url):
try:
client = BasicHttpClient(url)
get = GetMethod(url)
code = client.executeMethod(get)
body = get.getResponseBodyAsString().strip()
return (str(code), body)
except Exception, e:
self.log.error("Error during HTTP request: ", e)
return (None, None)
def urlGet(self, url):
try:
client = BasicHttpClient(url)
get = GetMethod(url)
code = client.executeMethod(get)
body = get.getResponseBodyAsString().strip()
return (str(code), body)
except Exception, e:
self.log.error("Error during HTTP request: ", e)
return (None, None)
def enhance( self, contentDescriptor, content, connector_url, cq_user, cq_password, asset_doWebDAV, asset_skeleton, asset_transformationRules, asset_drive, asset_filenameAttr, asset_pathAttr ):
# Validate Parameters
if not self.validInputs(connector_url,cq_user,cq_password,asset_drive,asset_filenameAttr,asset_pathAttr,"page_query","page_querylib","ia_path","ia_cacheCols"): return
if asset_doWebDAV:
# Save Asset to WebDAV Drive
drive = asset_drive[0]
path = re.sub("/content/dam/","/var/dam/",contentDescriptor.metadata[asset_pathAttr])
filename = contentDescriptor.metadata[asset_filenameAttr]
filepath = "%s:%s/%s" % (drive,path,filename)
if not exists("%s:%s" % (drive,path)): makedirs("%s:%s" % (drive,path))
file = BufferedOutputStream(FileOutputStream(filepath))
data = Base64.decode(content.contentData)
try:
file.write(data, 0, len(data))
file.close()
contentDescriptor.metadata["Load.Status"] = "SUCCESS"
except:
logger.error("! [WebDAV] Could not write %s to file system" % (contentDescriptor.url))
file.close()
contentDescriptor.metadata["Load.Status"] = "FAILURE"
contentDescriptor.metadata["Load.Failure Reason"] = "! [WebDAV] Could not deposit file on file system"
return
# Build & Send Asset Metadata Payload if required
if asset_skeleton and asset_transformationRules:
# Build Payload
payload = self.buildAssetPayload(asset_skeleton,asset_transformationRules,content,contentDescriptor)
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password))
header = Header("Authorization", "Basic %s" % (base64EncodedCredentials))
# Send Metadata Payload
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
[loadStatus, failureReason] = self.sendPayload(client,get,post,payload,contentDescriptor.url)
contentDescriptor.metadata["Load.Status"] = loadStatus
if loadStatus == "SUCCESS":
contentDescriptor.metadata["Load.Failure Reason"] = ''
else:
contentDescriptor.metadata["Load.Failure Reason"] = "Metadata: "+failureReason
logger.error("! [METADATA] %s" % (failureReason))
def notifyExternalRelationship(self, relationship,pid,system,identifier):
try:
url = self.systemConfig.getString(None, "curation","external-system-urls","notify-curation",system)
self.writer.println(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier)
client = BasicHttpClient(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier)
get = GetMethod(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier)
client.executeMethod(get)
status = get.getStatusCode()
if status != 200:
text = get.getStatusText()
self.log.error(String.format("Error accessing ReDBox: %s %s",status, text));
return None;
except Exception, ex:
return None;
def doHttpGet(url, timeout=20000, requestedData='body', headerName=None):
"""
Performs HTTP(S) Connection to the specified URL
Returns data according to the requestedData flag:
'body': Full Response Body as String
'header': Returns the response header with the specified headerName
"""
if requestedData == 'header':
method = HeadMethod(url)
else:
method = GetMethod(url)
client = HttpClient()
client.getHttpConnectionManager().getParams().setConnectionTimeout(timeout)
client.getHttpConnectionManager().getParams().setSoTimeout(timeout)
client.executeMethod(method)
if (requestedData == 'body'):
return method.getResponseBodyAsString()
elif (requestedData == 'header'):
if headerName:
return method.getResponseHeader(headerName)
else:
result = method.getResponseHeaders()
return ''.join([s.toString() for s in result])
else:
raise ValueError('Response part %s in not supported' % requestedData)
def notifyExternalRelationship(self,relationship,pid,system,identifier):
try:
url = self.systemConfig.getString(None, "curation","external-system-urls","notify-curation",system)
url = "http://localhost:9001/default/api/notifyCuration.script?apiKey=1412412412241"
completeUrl = url+ "&relationship=isCollectorOf&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString("redbox","system")+"&sourceIdentifier="+identifier
self.writer.println("the completeUrl: "+ completeUrl)
client = BasicHttpClient(completeUrl)
get = GetMethod(completeUrl)
client.executeMethod(get)
status = get.getStatusCode()
if status != 200:
text = get.getStatusText()
self.log.error(String.format("Error accessing Mint",status, text));
return None;
except Exception, ex:
return None;
def enhancePages(self, project, connector_url, cq_user, cq_password,
page_query, page_queryLib):
# Validate Parameters
if not self.validInputs(connector_url, cq_user, cq_password,
"asset_drive", "asset_filenameAttr",
"asset_pathAttr", page_query, page_queryLib,
"ia_path", "ia_cacheCols"):
return
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString("%s:%s" %
(cq_user, cq_password))
header = Header("Authorization",
"Basic %s" % (base64EncodedCredentials))
# Cycle through contentdescriptors specified by scope query
query = queryLibrarianService.findQueryByName(page_queryLib,
page_query)
params = HashMap()
params.put('projectId', project.id)
res = queryLibrarianService.executeQuery(query, params)
# Get cd
for r in res:
cd = r[0]
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
try:
payload = cd.getContent()[0].getContentData().encode("utf-8")
[loadStatus,
failureReason] = self.sendPayload(client, get, post, payload,
cd.url)
cd.metadata["Load.Status"] = loadStatus
cd.metadata["Load.Failure Reason"] = failureReason
if loadStatus != "SUCCESS":
logger.error("url: %s, reason: %s" %
(cd.url, failureReason))
except:
cd.metadata["Load.Status"] = "FAILURE"
cd.metadata["Load.Failure Reason"] = "Null content"
logger.error("url: %s, reason: %s" % (cd.url, failureReason))
def queryJobStatus(self, curationJob):
relations = ArrayList()
get = None
try:
url = self.systemConfig.getString(None, "curation","curation-manager-url")
client = BasicHttpClient(url + "/job/"+ curationJob.getCurationJobId())
get = GetMethod(url+ "/job/"+ curationJob.getCurationJobId())
client.executeMethod(get)
status = get.getStatusCode()
if status != 200:
text = get.getStatusText()
self.log.error(String.format("Error accessing Curation Manager, status code '%d' returned with message: %s",status, text));
return None;
except Exception, ex:
return None;
def __init__(self):
responseType = "text/html; charset=UTF-8"
responseMsg = ""
func = formData.get("func")
if func == "placeName":
try:
placeName = formData.get("q")
url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName
client = BasicHttpClient(url)
get = GetMethod(url)
statusInt = client.executeMethod(get)
r = str(statusInt)
jsonConfigHelper = JsonConfigHelper(get.getResponseBodyAsString().strip())
for geoName in jsonConfigHelper.getJsonList("geonames"):
responseMsg += "%s, %s|%s \n" % (geoName.get("name"), geoName.get("countryName"), geoName.get("geonameId"))
except Exception, e:
print "exception: ", str(e)
r = str(e), None
responseType = "text/plain; charset=UTF-8"
def enhanceIA(self, connector_url, cq_user, cq_password, ia_path,
ia_cacheCols):
# Validate Parameters
if not self.validInputs(connector_url, cq_user, cq_password,
"asset_drive", "asset_filenameAttr",
"asset_pathAttr", "page_query",
"page_queryLib", ia_path, ia_cacheCols):
return
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString("%s:%s" %
(cq_user, cq_password))
header = Header("Authorization",
"Basic %s" % (base64EncodedCredentials))
# IA Build
cached_IA = self.cacheIA(ia_path, ia_cacheCols)
for placeholder in cached_IA:
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
# Prep Payload
path = placeholder["Path"]
filename = placeholder["Filename"]
template_path = placeholder["Template Path"]
label = placeholder["Label"]
# Send Payload
payload = '<page path="%s" filename="%s" template_path="%s" label="%s"><node path="jcr:content" type="nt:unstructured"><prop name="jcr:title" type="String">%s</prop></node></page>' % (
path, filename, template_path, label, label)
[loadStatus,
failureReason] = self.sendPayload(client, get, post, payload,
path + filename)
if loadStatus != "SUCCESS":
logger.error("url: %s, reason: %s" %
(placeholder["URL"], failureReason))
def __activate__(self, context):
self.vc = context
responseType = "text/html; charset=UTF-8"
responseMsg = ""
func = self.vc("formData").get("func")
if func == "placeName":
try:
placeName = self.vc("formData").get("q")
url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName
client = BasicHttpClient(url)
get = GetMethod(url)
statusInt = client.executeMethod(get)
r = str(statusInt)
json = JsonSimple(get.getResponseBodyAsString().strip())
for geoName in json.getJsonSimpleList("geonames"):
responseMsg += "%s, %s|%s \n" % (geoName.getString(None, "name"), geoName.getString(None, "countryName"), geoName.getString(None, "geonameId"))
except Exception, e:
print "exception: ", str(e)
r = str(e), None
responseType = "text/plain; charset=UTF-8"
def saveHM(filename):
# Create new HTTP client
client = HttpClient()
client.getParams().setAuthenticationPreemptive(True)
defaultcreds = UsernamePasswordCredentials(user, password)
client.getState().setCredentials(AuthScope.ANY, defaultcreds)
# Get data across HTTP
url = (
"http://"
+ host
+ ":"
+ str(port)
+ "/admin/savedataview.egi?type="
+ type
+ "&data_format=NEXUSHDF5_LZW_6&data_saveopen_action=OPEN_ONLY"
)
getMethod = GetMethod(url)
getMethod.setDoAuthentication(True)
client.executeMethod(getMethod)
# Save locally
file = File(directory + "/" + filename)
out = FileOutputStream(file)
out.write(getMethod.getResponseBody())
out.close()
# Clean up
getMethod.releaseConnection()
def queryJobStatus(self, curationJob):
relations = ArrayList()
get = None
try:
url = self.systemConfig.getString(None, "curation",
"curation-manager-url")
client = BasicHttpClient(url + "/job/" +
curationJob.getCurationJobId())
get = GetMethod(url + "/job/" + curationJob.getCurationJobId())
client.executeMethod(get)
status = get.getStatusCode()
if status != 200:
text = get.getStatusText()
self.log.error(
String.format(
"Error accessing Curation Manager, status code '%d' returned with message: %s",
status, text))
return None
except Exception, ex:
return None
def notifyExternalRelationship(self, relationship, pid, system,
identifier):
try:
url = self.systemConfig.getString(None, "curation",
"external-system-urls",
"notify-curation", system)
url = "http://localhost:9001/default/api/notifyCuration.script?apiKey=1412412412241"
completeUrl = url + "&relationship=isCollectorOf&curatedPid=" + pid + "&identifier=" + relationship.get(
"identifier") + "&system=" + self.systemConfig.getString(
"redbox", "system") + "&sourceIdentifier=" + identifier
self.writer.println("the completeUrl: " + completeUrl)
client = BasicHttpClient(completeUrl)
get = GetMethod(completeUrl)
client.executeMethod(get)
status = get.getStatusCode()
if status != 200:
text = get.getStatusText()
self.log.error(
String.format("Error accessing Mint", status, text))
return None
except Exception, ex:
return None
def __activate__(self, context):
self.vc = context
responseType = "text/html; charset=UTF-8"
responseMsg = ""
func = self.vc("formData").get("func")
if func == "placeName":
try:
placeName = self.vc("formData").get("q")
url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName
client = BasicHttpClient(url)
get = GetMethod(url)
statusInt = client.executeMethod(get)
r = str(statusInt)
json = JsonSimple(get.getResponseBodyAsString().strip())
for geoName in json.getJsonSimpleList("geonames"):
responseMsg += "%s, %s|%s \n" % (geoName.getString(
None, "name"), geoName.getString(None, "countryName"),
geoName.getString(
None, "geonameId"))
except Exception, e:
print "exception: ", str(e)
r = str(e), None
responseType = "text/plain; charset=UTF-8"
def __wget(self, url):
client = BasicHttpClient(url)
m = GetMethod(url)
client.executeMethod(m)
return IOUtils.toString(m.getResponseBodyAsStream(), "UTF-8")
def enhance(self, contentDescriptor, content, connector_url, cq_user,
cq_password, asset_doWebDAV, asset_skeleton,
asset_transformationRules, asset_drive, asset_filenameAttr,
asset_pathAttr):
# Validate Parameters
if not self.validInputs(connector_url, cq_user, cq_password,
asset_drive, asset_filenameAttr,
asset_pathAttr, "page_query", "page_querylib",
"ia_path", "ia_cacheCols"):
return
if asset_doWebDAV:
# Save Asset to WebDAV Drive
drive = asset_drive[0]
path = re.sub("/content/dam/", "/var/dam/",
contentDescriptor.metadata[asset_pathAttr])
filename = contentDescriptor.metadata[asset_filenameAttr]
filepath = "%s:%s/%s" % (drive, path, filename)
if not exists("%s:%s" % (drive, path)):
makedirs("%s:%s" % (drive, path))
file = BufferedOutputStream(FileOutputStream(filepath))
data = Base64.decode(content.contentData)
try:
file.write(data, 0, len(data))
file.close()
contentDescriptor.metadata["Load.Status"] = "SUCCESS"
except:
logger.error("! [WebDAV] Could not write %s to file system" %
(contentDescriptor.url))
file.close()
contentDescriptor.metadata["Load.Status"] = "FAILURE"
contentDescriptor.metadata[
"Load.Failure Reason"] = "! [WebDAV] Could not deposit file on file system"
return
# Build & Send Asset Metadata Payload if required
if asset_skeleton and asset_transformationRules:
# Build Payload
payload = self.buildAssetPayload(asset_skeleton,
asset_transformationRules,
content, contentDescriptor)
# HTTP Connection Details
client = HttpClient()
base64EncodedCredentials = Base64.encodeString(
"%s:%s" % (cq_user, cq_password))
header = Header("Authorization",
"Basic %s" % (base64EncodedCredentials))
# Send Metadata Payload
post = PostMethod(connector_url)
post.addRequestHeader(header)
get = GetMethod(connector_url)
get.addRequestHeader(header)
[loadStatus,
failureReason] = self.sendPayload(client, get, post, payload,
contentDescriptor.url)
contentDescriptor.metadata["Load.Status"] = loadStatus
if loadStatus == "SUCCESS":
contentDescriptor.metadata["Load.Failure Reason"] = ''
else:
contentDescriptor.metadata[
"Load.Failure Reason"] = "Metadata: " + failureReason
logger.error("! [METADATA] %s" % (failureReason))
def __wget(self, url):
client = BasicHttpClient(url)
m = GetMethod(url)
client.executeMethod(m)
return IOUtils.toString(m.getResponseBodyAsStream(), "UTF-8")
import org.apache.commons.httpclient.*;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
public class Csdn {
@SuppressWarnings("deprecation")
public static void main(String[] args) {
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod("http://hi.csdn.net/gxiangzi");
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultMethodRetryHandler());
for (int i = 1; i <= 25; i++) {
try {
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.out.print("失败：" + getMethod.getStatusLine());
}
// byte[] responseBody=getMethod.getResponseBody();获取网页内容
// System.out.print(new String(responseBody));
} catch (Exception e) {
System.out.print("请检查网络地址！");
} finally {
getMethod.releaseConnection();
}
System.out.println("您已经成功刷新了"+i+"次！");
}
}
}
